aboutsummaryrefslogtreecommitdiffstats
path: root/python
diff options
context:
space:
mode:
authorJack Moffitt <jack@metajack.im>2014-08-28 09:34:23 -0600
committerJack Moffitt <jack@metajack.im>2014-09-08 20:21:42 -0600
commitc6ab60dbfc6da7b4f800c9e40893c8b58413960c (patch)
treed1d74076cf7fa20e4f77ec7cb82cae98b67362cb /python
parentdb2f642c32fc5bed445bb6f2e45b0f6f0b4342cf (diff)
downloadservo-c6ab60dbfc6da7b4f800c9e40893c8b58413960c.tar.gz
servo-c6ab60dbfc6da7b4f800c9e40893c8b58413960c.zip
Cargoify servo
Diffstat (limited to 'python')
-rw-r--r--python/licenseck.py75
-rw-r--r--python/mach/README.rst328
-rw-r--r--python/mach/bash-completion.sh29
-rw-r--r--python/mach/mach/__init__.py0
-rw-r--r--python/mach/mach/base.py110
-rw-r--r--python/mach/mach/commands/__init__.py0
-rw-r--r--python/mach/mach/commands/commandinfo.py41
-rw-r--r--python/mach/mach/commands/settings.py50
-rw-r--r--python/mach/mach/config.py488
-rw-r--r--python/mach/mach/decorators.py176
-rw-r--r--python/mach/mach/dispatcher.py277
-rw-r--r--python/mach/mach/logging.py256
-rw-r--r--python/mach/mach/main.py615
-rw-r--r--python/mach/mach/mixin/__init__.py0
-rw-r--r--python/mach/mach/mixin/logging.py55
-rw-r--r--python/mach/mach/mixin/process.py175
-rw-r--r--python/mach/mach/registrar.py65
-rw-r--r--python/mach/mach/terminal.py75
-rw-r--r--python/mach/mach/test/__init__.py0
-rw-r--r--python/mach/mach/test/common.py40
-rw-r--r--python/mach/mach/test/providers/__init__.py0
-rw-r--r--python/mach/mach/test/providers/basic.py15
-rw-r--r--python/mach/mach/test/providers/conditions.py53
-rw-r--r--python/mach/mach/test/providers/conditions_invalid.py16
-rw-r--r--python/mach/mach/test/providers/throw.py29
-rw-r--r--python/mach/mach/test/providers/throw2.py13
-rw-r--r--python/mach/mach/test/test_conditions.py82
-rw-r--r--python/mach/mach/test/test_config.py264
-rw-r--r--python/mach/mach/test/test_entry_point.py60
-rw-r--r--python/mach/mach/test/test_error_output.py39
-rw-r--r--python/mach/mach/test/test_logger.py47
-rw-r--r--python/mach/setup.py38
-rw-r--r--python/mach_bootstrap.py100
-rw-r--r--python/servo/__init__.py0
-rw-r--r--python/servo/bootstrap_commands.py153
-rw-r--r--python/servo/build_commands.py85
-rw-r--r--python/servo/command_base.py94
-rw-r--r--python/servo/devenv_commands.py32
-rw-r--r--python/servo/post_build_commands.py44
-rw-r--r--python/servo/testing_commands.py122
-rw-r--r--python/tidy.py91
-rw-r--r--python/toml/LICENSE21
-rw-r--r--python/toml/PKG-INFO52
-rw-r--r--python/toml/README.rst42
-rw-r--r--python/toml/setup.py14
-rw-r--r--python/toml/toml.py443
46 files changed, 4804 insertions, 0 deletions
diff --git a/python/licenseck.py b/python/licenseck.py
new file mode 100644
index 00000000000..0e862e35ef4
--- /dev/null
+++ b/python/licenseck.py
@@ -0,0 +1,75 @@
+# Copyright 2013 The Servo Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+license0="""\
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"""
+
+license1="""\
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+
+license2="""\
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+
+license3 = """\
+// Copyright 2013 The Servo Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+"""
+
+license4 = """\
+# Copyright 2013 The Servo Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+"""
+
+licenses = [license0, license1, license2, license3, license4]
+
+exceptions = [
+ "servo/dom/bindings/codegen/ply/ply/yacc.py", # BSD
+ "servo/dom/bindings/codegen/ply/ply/__init__.py", # BSD
+ "servo/dom/bindings/codegen/ply/ply/lex.py", # BSD
+]
+
+def check_license(name, contents):
+ valid_license = False
+ for a_valid_license in licenses:
+ if contents.startswith(a_valid_license):
+ valid_license = True
+ break
+ if valid_license:
+ return True
+
+ for exception in exceptions:
+ if name.endswith(exception):
+ return True
+
+ firstlineish = contents[:100]
+ if firstlineish.find("xfail-license") != -1:
+ return True
+
+ return False
diff --git a/python/mach/README.rst b/python/mach/README.rst
new file mode 100644
index 00000000000..25e8fd470bc
--- /dev/null
+++ b/python/mach/README.rst
@@ -0,0 +1,328 @@
+====
+mach
+====
+
+Mach (German for *do*) is a generic command dispatcher for the command
+line.
+
+To use mach, you install the mach core (a Python package), create an
+executable *driver* script (named whatever you want), and write mach
+commands. When the *driver* is executed, mach dispatches to the
+requested command handler automatically.
+
+Features
+========
+
+On a high level, mach is similar to using argparse with subparsers (for
+command handling). When you dig deeper, mach offers a number of
+additional features:
+
+Distributed command definitions
+ With optparse/argparse, you have to define your commands on a central
+ parser instance. With mach, you annotate your command methods with
+ decorators and mach finds and dispatches to them automatically.
+
+Command categories
+ Mach commands can be grouped into categories when displayed in help.
+ This is currently not possible with argparse.
+
+Logging management
+ Mach provides a facility for logging (both classical text and
+ structured) that is available to any command handler.
+
+Settings files
+ Mach provides a facility for reading settings from an ini-like file
+ format.
+
+Components
+==========
+
+Mach is conceptually composed of the following components:
+
+core
+ The mach core is the core code powering mach. This is a Python package
+ that contains all the business logic that makes mach work. The mach
+ core is common to all mach deployments.
+
+commands
+ These are what mach dispatches to. Commands are simply Python methods
+ registered as command names. The set of commands is unique to the
+ environment mach is deployed in.
+
+driver
+ The *driver* is the entry-point to mach. It is simply an executable
+ script that loads the mach core, tells it where commands can be found,
+ then asks the mach core to handle the current request. The driver is
+ unique to the deployed environment. But, it's usually based on an
+ example from this source tree.
+
+Project State
+=============
+
+mach was originally written as a command dispatching framework to aid
+Firefox development. While the code is mostly generic, there are still
+some pieces that closely tie it to Mozilla/Firefox. The goal is for
+these to eventually be removed and replaced with generic features so
+mach is suitable for anybody to use. Until then, mach may not be the
+best fit for you.
+
+Implementing Commands
+---------------------
+
+Mach commands are defined via Python decorators.
+
+All the relevant decorators are defined in the *mach.decorators* module.
+The important decorators are as follows:
+
+CommandProvider
+ A class decorator that denotes that a class contains mach
+ commands. The decorator takes no arguments.
+
+Command
+ A method decorator that denotes that the method should be called when
+ the specified command is requested. The decorator takes a command name
+ as its first argument and a number of additional arguments to
+ configure the behavior of the command.
+
+CommandArgument
+ A method decorator that defines an argument to the command. Its
+ arguments are essentially proxied to ArgumentParser.add_argument()
+
+Classes with the *@CommandProvider* decorator *must* have an *__init__*
+method that accepts 1 or 2 arguments. If it accepts 2 arguments, the
+2nd argument will be a *MachCommandContext* instance. This is just a named
+tuple containing references to objects provided by the mach driver.
+
+Here is a complete example::
+
+ from mach.decorators import (
+ CommandArgument,
+ CommandProvider,
+ Command,
+ )
+
+ @CommandProvider
+ class MyClass(object):
+ @Command('doit', help='Do ALL OF THE THINGS.')
+ @CommandArgument('--force', '-f', action='store_true',
+ help='Force doing it.')
+ def doit(self, force=False):
+ # Do stuff here.
+
+When the module is loaded, the decorators tell mach about all handlers.
+When mach runs, it takes the assembled metadata from these handlers and
+hooks it up to the command line driver. Under the hood, arguments passed
+to the decorators are being used to help mach parse command arguments,
+formulate arguments to the methods, etc. See the documentation in the
+*mach.base* module for more.
+
+The Python modules defining mach commands do not need to live inside the
+main mach source tree.
+
+Conditionally Filtering Commands
+--------------------------------
+
+Sometimes it might only make sense to run a command given a certain
+context. For example, running tests only makes sense if the product
+they are testing has been built, and said build is available. To make
+sure a command is only runnable from within a correct context, you can
+define a series of conditions on the *Command* decorator.
+
+A condition is simply a function that takes an instance of the
+*CommandProvider* class as an argument, and returns True or False. If
+any of the conditions defined on a command return False, the command
+will not be runnable. The doc string of a condition function is used in
+error messages, to explain why the command cannot currently be run.
+
+Here is an example:
+
+ from mach.decorators import (
+ CommandProvider,
+ Command,
+ )
+
+ def build_available(cls):
+ """The build needs to be available."""
+ return cls.build_path is not None
+
+ @CommandProvider
+ class MyClass(MachCommandBase):
+ def __init__(self, build_path=None):
+ self.build_path = build_path
+
+ @Command('run_tests', conditions=[build_available])
+ def run_tests(self):
+ # Do stuff here.
+
+It is important to make sure that any state needed by the condition is
+available to instances of the command provider.
+
+By default all commands without any conditions applied will be runnable,
+but it is possible to change this behaviour by setting *require_conditions*
+to True:
+
+ m = mach.main.Mach()
+ m.require_conditions = True
+
+Minimizing Code in Commands
+---------------------------
+
+Mach command modules, classes, and methods work best when they are
+minimal dispatchers. The reason is import bloat. Currently, the mach
+core needs to import every Python file potentially containing mach
+commands for every command invocation. If you have dozens of commands or
+commands in modules that import a lot of Python code, these imports
+could slow mach down and waste memory.
+
+It is thus recommended that mach modules, classes, and methods do as
+little work as possible. Ideally the module should only import from
+the *mach* package. If you need external modules, you should import them
+from within the command method.
+
+To keep code size small, the body of a command method should be limited
+to:
+
+1. Obtaining user input (parsing arguments, prompting, etc)
+2. Calling into some other Python package
+3. Formatting output
+
+Of course, these recommendations can be ignored if you want to risk
+slower performance.
+
+In the future, the mach driver may cache the dispatching information or
+have it intelligently loaded to facilitate lazy loading.
+
+Logging
+=======
+
+Mach configures a built-in logging facility so commands can easily log
+data.
+
+What sets the logging facility apart from most loggers you've seen is
+that it encourages structured logging. Instead of conventional logging
+where simple strings are logged, the internal logging mechanism logs all
+events with the following pieces of information:
+
+* A string *action*
+* A dict of log message fields
+* A formatting string
+
+Essentially, instead of assembling a human-readable string at
+logging-time, you create an object holding all the pieces of data that
+will constitute your logged event. For each unique type of logged event,
+you assign an *action* name.
+
+Depending on how logging is configured, your logged event could get
+written a couple of different ways.
+
+JSON Logging
+------------
+
+Where machines are the intended target of the logging data, a JSON
+logger is configured. The JSON logger assembles an array consisting of
+the following elements:
+
+* Decimal wall clock time in seconds since UNIX epoch
+* String *action* of message
+* Object with structured message data
+
+The JSON-serialized array is written to a configured file handle.
+Consumers of this logging stream can just perform a readline() then feed
+that into a JSON deserializer to reconstruct the original logged
+message. They can key off the *action* element to determine how to
+process individual events. There is no need to invent a parser.
+Convenient, isn't it?
+
+Logging for Humans
+------------------
+
+Where humans are the intended consumer of a log message, the structured
+log message are converted to more human-friendly form. This is done by
+utilizing the *formatting* string provided at log time. The logger
+simply calls the *format* method of the formatting string, passing the
+dict containing the message's fields.
+
+When *mach* is used in a terminal that supports it, the logging facility
+also supports terminal features such as colorization. This is done
+automatically in the logging layer - there is no need to control this at
+logging time.
+
+In addition, messages intended for humans typically prepends every line
+with the time passed since the application started.
+
+Logging HOWTO
+-------------
+
+Structured logging piggybacks on top of Python's built-in logging
+infrastructure provided by the *logging* package. We accomplish this by
+taking advantage of *logging.Logger.log()*'s *extra* argument. To this
+argument, we pass a dict with the fields *action* and *params*. These
+are the string *action* and dict of message fields, respectively. The
+formatting string is passed as the *msg* argument, like normal.
+
+If you were logging to a logger directly, you would do something like:
+
+ logger.log(logging.INFO, 'My name is {name}',
+ extra={'action': 'my_name', 'params': {'name': 'Gregory'}})
+
+The JSON logging would produce something like:
+
+ [1339985554.306338, "my_name", {"name": "Gregory"}]
+
+Human logging would produce something like:
+
+ 0.52 My name is Gregory
+
+Since there is a lot of complexity using logger.log directly, it is
+recommended to go through a wrapping layer that hides part of the
+complexity for you. The easiest way to do this is by utilizing the
+LoggingMixin:
+
+ import logging
+ from mach.mixin.logging import LoggingMixin
+
+ class MyClass(LoggingMixin):
+ def foo(self):
+ self.log(logging.INFO, 'foo_start', {'bar': True},
+ 'Foo performed. Bar: {bar}')
+
+Entry Points
+============
+
+It is possible to use setuptools' entry points to load commands
+directly from python packages. A mach entry point is a function which
+returns a list of files or directories containing mach command
+providers. e.g.::
+
+ def list_providers():
+ providers = []
+ here = os.path.abspath(os.path.dirname(__file__))
+ for p in os.listdir(here):
+ if p.endswith('.py'):
+ providers.append(os.path.join(here, p))
+ return providers
+
+See http://pythonhosted.org/setuptools/setuptools.html#dynamic-discovery-of-services-and-plugins
+for more information on creating an entry point. To search for entry
+point plugins, you can call *load_commands_from_entry_point*. This
+takes a single parameter called *group*. This is the name of the entry
+point group to load and defaults to ``mach.providers``. e.g.::
+
+ mach.load_commands_from_entry_point("mach.external.providers")
+
+Adding Global Arguments
+=======================
+
+Arguments to mach commands are usually command-specific. However,
+mach ships with a handful of global arguments that apply to all
+commands.
+
+It is possible to extend the list of global arguments. In your
+*mach driver*, simply call ``add_global_argument()`` on your
+``mach.main.Mach`` instance. e.g.::
+
+ mach = mach.main.Mach(os.getcwd())
+
+ # Will allow --example to be specified on every mach command.
+ mach.add_global_argument('--example', action='store_true',
+ help='Demonstrate an example global argument.')
diff --git a/python/mach/bash-completion.sh b/python/mach/bash-completion.sh
new file mode 100644
index 00000000000..e4b151f24c9
--- /dev/null
+++ b/python/mach/bash-completion.sh
@@ -0,0 +1,29 @@
+function _mach()
+{
+ local cur cmds c subcommand
+ COMPREPLY=()
+
+ # Load the list of commands
+ cmds=`"${COMP_WORDS[0]}" mach-commands`
+
+ # Look for the subcommand.
+ cur="${COMP_WORDS[COMP_CWORD]}"
+ subcommand=""
+ c=1
+ while [ $c -lt $COMP_CWORD ]; do
+ word="${COMP_WORDS[c]}"
+ for cmd in $cmds; do
+ if [ "$cmd" = "$word" ]; then
+ subcommand="$word"
+ fi
+ done
+ c=$((++c))
+ done
+
+ if [[ "$subcommand" == "help" || -z "$subcommand" ]]; then
+ COMPREPLY=( $(compgen -W "$cmds" -- ${cur}) )
+ fi
+
+ return 0
+}
+complete -o default -F _mach mach
diff --git a/python/mach/mach/__init__.py b/python/mach/mach/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/python/mach/mach/__init__.py
diff --git a/python/mach/mach/base.py b/python/mach/mach/base.py
new file mode 100644
index 00000000000..3e8e6477357
--- /dev/null
+++ b/python/mach/mach/base.py
@@ -0,0 +1,110 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+
+class CommandContext(object):
+ """Holds run-time state so it can easily be passed to command providers."""
+ def __init__(self, cwd=None, settings=None, log_manager=None,
+ commands=None, **kwargs):
+ self.cwd = cwd
+ self.settings = settings
+ self.log_manager = log_manager
+ self.commands = commands
+
+ for k,v in kwargs.items():
+ setattr(self, k, v)
+
+
+class MachError(Exception):
+ """Base class for all errors raised by mach itself."""
+
+
+class NoCommandError(MachError):
+ """No command was passed into mach."""
+
+
+class UnknownCommandError(MachError):
+ """Raised when we attempted to execute an unknown command."""
+
+ def __init__(self, command, verb, suggested_commands=None):
+ MachError.__init__(self)
+
+ self.command = command
+ self.verb = verb
+ self.suggested_commands = suggested_commands or []
+
+class UnrecognizedArgumentError(MachError):
+ """Raised when an unknown argument is passed to mach."""
+
+ def __init__(self, command, arguments):
+ MachError.__init__(self)
+
+ self.command = command
+ self.arguments = arguments
+
+
+class MethodHandler(object):
+ """Describes a Python method that implements a mach command.
+
+ Instances of these are produced by mach when it processes classes
+ defining mach commands.
+ """
+ __slots__ = (
+ # The Python class providing the command. This is the class type not
+ # an instance of the class. Mach will instantiate a new instance of
+ # the class if the command is executed.
+ 'cls',
+
+ # Whether the __init__ method of the class should receive a mach
+ # context instance. This should only affect the mach driver and how
+ # it instantiates classes.
+ 'pass_context',
+
+ # The name of the method providing the command. In other words, this
+ # is the str name of the attribute on the class type corresponding to
+ # the name of the function.
+ 'method',
+
+ # The name of the command.
+ 'name',
+
+ # String category this command belongs to.
+ 'category',
+
+ # Description of the purpose of this command.
+ 'description',
+
+ # Whether to allow all arguments from the parser.
+ 'allow_all_arguments',
+
+ # Functions used to 'skip' commands if they don't meet the conditions
+ # in a given context.
+ 'conditions',
+
+ # argparse.ArgumentParser instance to use as the basis for command
+ # arguments.
+ 'parser',
+
+ # Arguments added to this command's parser. This is a 2-tuple of
+ # positional and named arguments, respectively.
+ 'arguments',
+ )
+
+ def __init__(self, cls, method, name, category=None, description=None,
+ allow_all_arguments=False, conditions=None, parser=None, arguments=None,
+ pass_context=False):
+
+ self.cls = cls
+ self.method = method
+ self.name = name
+ self.category = category
+ self.description = description
+ self.allow_all_arguments = allow_all_arguments
+ self.conditions = conditions or []
+ self.parser = parser
+ self.arguments = arguments or []
+ self.pass_context = pass_context
+
diff --git a/python/mach/mach/commands/__init__.py b/python/mach/mach/commands/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/python/mach/mach/commands/__init__.py
diff --git a/python/mach/mach/commands/commandinfo.py b/python/mach/mach/commands/commandinfo.py
new file mode 100644
index 00000000000..3cca0af202e
--- /dev/null
+++ b/python/mach/mach/commands/commandinfo.py
@@ -0,0 +1,41 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, # You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import print_function, unicode_literals
+
+from mach.decorators import (
+ CommandProvider,
+ Command,
+)
+
+
+@CommandProvider
+class BuiltinCommands(object):
+ def __init__(self, context):
+ self.context = context
+
+ @Command('mach-commands', category='misc',
+ description='List all mach commands.')
+ def commands(self):
+ print("\n".join(self.context.commands.command_handlers.keys()))
+
+ @Command('mach-debug-commands', category='misc',
+ description='Show info about available mach commands.')
+ def debug_commands(self):
+ import inspect
+
+ handlers = self.context.commands.command_handlers
+ for command in sorted(handlers.keys()):
+ handler = handlers[command]
+ cls = handler.cls
+ method = getattr(cls, getattr(handler, 'method'))
+
+ print(command)
+ print('=' * len(command))
+ print('')
+ print('File: %s' % inspect.getsourcefile(method))
+ print('Class: %s' % cls.__name__)
+ print('Method: %s' % handler.method)
+ print('')
+
diff --git a/python/mach/mach/commands/settings.py b/python/mach/mach/commands/settings.py
new file mode 100644
index 00000000000..7223f241fb2
--- /dev/null
+++ b/python/mach/mach/commands/settings.py
@@ -0,0 +1,50 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import print_function, unicode_literals
+
+from textwrap import TextWrapper
+
+from mach.decorators import (
+ CommandProvider,
+ Command,
+)
+
+
+#@CommandProvider
+class Settings(object):
+ """Interact with settings for mach.
+
+ Currently, we only provide functionality to view what settings are
+ available. In the future, this module will be used to modify settings, help
+ people create configs via a wizard, etc.
+ """
+ def __init__(self, context):
+ self.settings = context.settings
+
+ @Command('settings-list', category='devenv',
+ description='Show available config settings.')
+ def list_settings(self):
+ """List available settings in a concise list."""
+ for section in sorted(self.settings):
+ for option in sorted(self.settings[section]):
+ short, full = self.settings.option_help(section, option)
+ print('%s.%s -- %s' % (section, option, short))
+
+ @Command('settings-create', category='devenv',
+ description='Print a new settings file with usage info.')
+ def create(self):
+ """Create an empty settings file with full documentation."""
+ wrapper = TextWrapper(initial_indent='# ', subsequent_indent='# ')
+
+ for section in sorted(self.settings):
+ print('[%s]' % section)
+ print('')
+
+ for option in sorted(self.settings[section]):
+ short, full = self.settings.option_help(section, option)
+
+ print(wrapper.fill(full))
+ print(';%s =' % option)
+ print('')
diff --git a/python/mach/mach/config.py b/python/mach/mach/config.py
new file mode 100644
index 00000000000..89824d554d0
--- /dev/null
+++ b/python/mach/mach/config.py
@@ -0,0 +1,488 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+r"""
+This file defines classes for representing config data/settings.
+
+Config data is modeled as key-value pairs. Keys are grouped together into named
+sections. Individual config settings (options) have metadata associated with
+them. This metadata includes type, default value, valid values, etc.
+
+The main interface to config data is the ConfigSettings class. 1 or more
+ConfigProvider classes are associated with ConfigSettings and define what
+settings are available.
+
+Descriptions of individual config options can be translated to multiple
+languages using gettext. Each option has associated with it a domain and locale
+directory. By default, the domain is the section the option is in and the
+locale directory is the "locale" directory beneath the directory containing the
+module that defines it.
+
+People implementing ConfigProvider instances are expected to define a complete
+gettext .po and .mo file for the en-US locale. You can use the gettext-provided
+msgfmt binary to perform this conversion. Generation of the original .po file
+can be done via the write_pot() of ConfigSettings.
+"""
+
+from __future__ import unicode_literals
+
+import collections
+import gettext
+import os
+import sys
+
+if sys.version_info[0] == 3:
+ from configparser import RawConfigParser
+ str_type = str
+else:
+ from ConfigParser import RawConfigParser
+ str_type = basestring
+
+
+class ConfigType(object):
+ """Abstract base class for config values."""
+
+ @staticmethod
+ def validate(value):
+ """Validates a Python value conforms to this type.
+
+ Raises a TypeError or ValueError if it doesn't conform. Does not do
+ anything if the value is valid.
+ """
+
+ @staticmethod
+ def from_config(config, section, option):
+ """Obtain the value of this type from a RawConfigParser.
+
+ Receives a RawConfigParser instance, a str section name, and the str
+ option in that section to retrieve.
+
+ The implementation may assume the option exists in the RawConfigParser
+ instance.
+
+ Implementations are not expected to validate the value. But, they
+ should return the appropriate Python type.
+ """
+
+ @staticmethod
+ def to_config(value):
+ return value
+
+
+class StringType(ConfigType):
+ @staticmethod
+ def validate(value):
+ if not isinstance(value, str_type):
+ raise TypeError()
+
+ @staticmethod
+ def from_config(config, section, option):
+ return config.get(section, option)
+
+
+class BooleanType(ConfigType):
+ @staticmethod
+ def validate(value):
+ if not isinstance(value, bool):
+ raise TypeError()
+
+ @staticmethod
+ def from_config(config, section, option):
+ return config.getboolean(section, option)
+
+ @staticmethod
+ def to_config(value):
+ return 'true' if value else 'false'
+
+
+class IntegerType(ConfigType):
+ @staticmethod
+ def validate(value):
+ if not isinstance(value, int):
+ raise TypeError()
+
+ @staticmethod
+ def from_config(config, section, option):
+ return config.getint(section, option)
+
+
+class PositiveIntegerType(IntegerType):
+ @staticmethod
+ def validate(value):
+ if not isinstance(value, int):
+ raise TypeError()
+
+ if value < 0:
+ raise ValueError()
+
+
+class PathType(StringType):
+ @staticmethod
+ def validate(value):
+ if not isinstance(value, str_type):
+ raise TypeError()
+
+ @staticmethod
+ def from_config(config, section, option):
+ return config.get(section, option)
+
+
+class AbsolutePathType(PathType):
+ @staticmethod
+ def validate(value):
+ if not isinstance(value, str_type):
+ raise TypeError()
+
+ if not os.path.isabs(value):
+ raise ValueError()
+
+
+class RelativePathType(PathType):
+ @staticmethod
+ def validate(value):
+ if not isinstance(value, str_type):
+ raise TypeError()
+
+ if os.path.isabs(value):
+ raise ValueError()
+
+
+class DefaultValue(object):
+ pass
+
+
+class ConfigProvider(object):
+ """Abstract base class for an object providing config settings.
+
+ Classes implementing this interface expose configurable settings. Settings
+ are typically only relevant to that component itself. But, nothing says
+ settings can't be shared by multiple components.
+ """
+
+ @classmethod
+ def register_settings(cls):
+ """Registers config settings.
+
+ This is called automatically. Child classes should likely not touch it.
+ See _register_settings() instead.
+ """
+ if hasattr(cls, '_settings_registered'):
+ return
+
+ cls._settings_registered = True
+
+ cls.config_settings = {}
+
+ ourdir = os.path.dirname(__file__)
+ cls.config_settings_locale_directory = os.path.join(ourdir, 'locale')
+
+ cls._register_settings()
+
+ @classmethod
+ def _register_settings(cls):
+ """The actual implementation of register_settings().
+
+ This is what child classes should implement. They should not touch
+ register_settings().
+
+ Implementations typically make 1 or more calls to _register_setting().
+ """
+ raise NotImplemented('%s must implement _register_settings.' %
+ __name__)
+
+ @classmethod
+ def register_setting(cls, section, option, type_cls, default=DefaultValue,
+ choices=None, domain=None):
+ """Register a config setting with this type.
+
+ This is a convenience method to populate available settings. It is
+ typically called in the class's _register_settings() implementation.
+
+ Each setting must have:
+
+ section -- str section to which the setting belongs. This is how
+ settings are grouped.
+
+ option -- str id for the setting. This must be unique within the
+ section it appears.
+
+ type -- a ConfigType-derived type defining the type of the setting.
+
+ Each setting has the following optional parameters:
+
+ default -- The default value for the setting. If None (the default)
+ there is no default.
+
+ choices -- A set of values this setting can hold. Values not in
+ this set are invalid.
+
+ domain -- Translation domain for this setting. By default, the
+ domain is the same as the section name.
+ """
+ if not section in cls.config_settings:
+ cls.config_settings[section] = {}
+
+ if option in cls.config_settings[section]:
+ raise Exception('Setting has already been registered: %s.%s' % (
+ section, option))
+
+ domain = domain if domain is not None else section
+
+ meta = {
+ 'short': '%s.short' % option,
+ 'full': '%s.full' % option,
+ 'type_cls': type_cls,
+ 'domain': domain,
+ 'localedir': cls.config_settings_locale_directory,
+ }
+
+ if default != DefaultValue:
+ meta['default'] = default
+
+ if choices is not None:
+ meta['choices'] = choices
+
+ cls.config_settings[section][option] = meta
+
+
+class ConfigSettings(collections.Mapping):
+ """Interface for configuration settings.
+
+ This is the main interface to the configuration.
+
+ A configuration is a collection of sections. Each section contains
+ key-value pairs.
+
+ When an instance is created, the caller first registers ConfigProvider
+ instances with it. This tells the ConfigSettings what individual settings
+ are available and defines extra metadata associated with those settings.
+ This is used for validation, etc.
+
+ Once ConfigProvider instances are registered, a config is populated. It can
+ be loaded from files or populated by hand.
+
+ ConfigSettings instances are accessed like dictionaries or by using
+ attributes. e.g. the section "foo" is accessed through either
+ settings.foo or settings['foo'].
+
+ Sections are modeled by the ConfigSection class which is defined inside
+ this one. They look just like dicts or classes with attributes. To access
+ the "bar" option in the "foo" section:
+
+ value = settings.foo.bar
+ value = settings['foo']['bar']
+ value = settings.foo['bar']
+
+ Assignment is similar:
+
+ settings.foo.bar = value
+ settings['foo']['bar'] = value
+ settings['foo'].bar = value
+
+ You can even delete user-assigned values:
+
+ del settings.foo.bar
+ del settings['foo']['bar']
+
+ If there is a default, it will be returned.
+
+ When settings are mutated, they are validated against the registered
+ providers. Setting unknown settings or setting values to illegal values
+ will result in exceptions being raised.
+ """
+
+ class ConfigSection(collections.MutableMapping, object):
+ """Represents an individual config section."""
+ def __init__(self, config, name, settings):
+ object.__setattr__(self, '_config', config)
+ object.__setattr__(self, '_name', name)
+ object.__setattr__(self, '_settings', settings)
+
+ # MutableMapping interface
+ def __len__(self):
+ return len(self._settings)
+
+ def __iter__(self):
+ return iter(self._settings.keys())
+
+ def __contains__(self, k):
+ return k in self._settings
+
+ def __getitem__(self, k):
+ if k not in self._settings:
+ raise KeyError('Option not registered with provider: %s' % k)
+
+ meta = self._settings[k]
+
+ if self._config.has_option(self._name, k):
+ return meta['type_cls'].from_config(self._config, self._name, k)
+
+ if not 'default' in meta:
+ raise KeyError('No default value registered: %s' % k)
+
+ return meta['default']
+
+ def __setitem__(self, k, v):
+ if k not in self._settings:
+ raise KeyError('Option not registered with provider: %s' % k)
+
+ meta = self._settings[k]
+
+ meta['type_cls'].validate(v)
+
+ if not self._config.has_section(self._name):
+ self._config.add_section(self._name)
+
+ self._config.set(self._name, k, meta['type_cls'].to_config(v))
+
+ def __delitem__(self, k):
+ self._config.remove_option(self._name, k)
+
+ # Prune empty sections.
+ if not len(self._config.options(self._name)):
+ self._config.remove_section(self._name)
+
+ def __getattr__(self, k):
+ return self.__getitem__(k)
+
+ def __setattr__(self, k, v):
+ self.__setitem__(k, v)
+
+ def __delattr__(self, k):
+ self.__delitem__(k)
+
+
+ def __init__(self):
+ self._config = RawConfigParser()
+
+ self._settings = {}
+ self._sections = {}
+ self._finalized = False
+ self._loaded_filenames = set()
+
+ def load_file(self, filename):
+ self.load_files([filename])
+
+ def load_files(self, filenames):
+ """Load a config from files specified by their paths.
+
+ Files are loaded in the order given. Subsequent files will overwrite
+ values from previous files. If a file does not exist, it will be
+ ignored.
+ """
+ filtered = [f for f in filenames if os.path.exists(f)]
+
+ fps = [open(f, 'rt') for f in filtered]
+ self.load_fps(fps)
+ self._loaded_filenames.update(set(filtered))
+ for fp in fps:
+ fp.close()
+
+ def load_fps(self, fps):
+ """Load config data by reading file objects."""
+
+ for fp in fps:
+ self._config.readfp(fp)
+
+ def loaded_files(self):
+ return self._loaded_filenames
+
+ def write(self, fh):
+ """Write the config to a file object."""
+ self._config.write(fh)
+
+ def validate(self):
+ """Ensure that the current config passes validation.
+
+ This is a generator of tuples describing any validation errors. The
+ elements of the tuple are:
+
+ (bool) True if error is fatal. False if just a warning.
+ (str) Type of validation issue. Can be one of ('unknown-section',
+ 'missing-required', 'type-error')
+ """
+
+ def register_provider(self, provider):
+ """Register a ConfigProvider with this settings interface."""
+
+ if self._finalized:
+ raise Exception('Providers cannot be registered after finalized.')
+
+ provider.register_settings()
+
+ for section_name, settings in provider.config_settings.items():
+ section = self._settings.get(section_name, {})
+
+ for k, v in settings.items():
+ if k in section:
+ raise Exception('Setting already registered: %s.%s' %
+ section_name, k)
+
+ section[k] = v
+
+ self._settings[section_name] = section
+
+ def write_pot(self, fh):
+ """Write a pot gettext translation file."""
+
+ for section in sorted(self):
+ fh.write('# Section %s\n\n' % section)
+ for option in sorted(self[section]):
+ fh.write('msgid "%s.%s.short"\n' % (section, option))
+ fh.write('msgstr ""\n\n')
+
+ fh.write('msgid "%s.%s.full"\n' % (section, option))
+ fh.write('msgstr ""\n\n')
+
+ fh.write('# End of section %s\n\n' % section)
+
+ def option_help(self, section, option):
+ """Obtain the translated help messages for an option."""
+
+ meta = self[section]._settings[option]
+
+ # Providers should always have an en-US translation. If they don't,
+ # they are coded wrong and this will raise.
+ default = gettext.translation(meta['domain'], meta['localedir'],
+ ['en-US'])
+
+ t = gettext.translation(meta['domain'], meta['localedir'],
+ fallback=True)
+ t.add_fallback(default)
+
+ short = t.ugettext('%s.%s.short' % (section, option))
+ full = t.ugettext('%s.%s.full' % (section, option))
+
+ return (short, full)
+
+ def _finalize(self):
+ if self._finalized:
+ return
+
+ for section, settings in self._settings.items():
+ s = ConfigSettings.ConfigSection(self._config, section, settings)
+ self._sections[section] = s
+
+ self._finalized = True
+
+ # Mapping interface.
+ def __len__(self):
+ return len(self._settings)
+
+ def __iter__(self):
+ self._finalize()
+
+ return iter(self._sections.keys())
+
+ def __contains__(self, k):
+ return k in self._settings
+
+ def __getitem__(self, k):
+ self._finalize()
+
+ return self._sections[k]
+
+ # Allow attribute access because it looks nice.
+ def __getattr__(self, k):
+ return self.__getitem__(k)
diff --git a/python/mach/mach/decorators.py b/python/mach/mach/decorators.py
new file mode 100644
index 00000000000..cb9a67428e7
--- /dev/null
+++ b/python/mach/mach/decorators.py
@@ -0,0 +1,176 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import collections
+import inspect
+import types
+
+from .base import (
+ MachError,
+ MethodHandler
+)
+
+from .config import ConfigProvider
+from .registrar import Registrar
+
+
+def CommandProvider(cls):
+ """Class decorator to denote that it provides subcommands for Mach.
+
+ When this decorator is present, mach looks for commands being defined by
+ methods inside the class.
+ """
+
+ # The implementation of this decorator relies on the parse-time behavior of
+ # decorators. When the module is imported, the method decorators (like
+ # @Command and @CommandArgument) are called *before* this class decorator.
+ # The side-effect of the method decorators is to store specifically-named
+ # attributes on the function types. We just scan over all functions in the
+ # class looking for the side-effects of the method decorators.
+
+ # Tell mach driver whether to pass context argument to __init__.
+ pass_context = False
+
+ if inspect.ismethod(cls.__init__):
+ spec = inspect.getargspec(cls.__init__)
+
+ if len(spec.args) > 2:
+ msg = 'Mach @CommandProvider class %s implemented incorrectly. ' + \
+ '__init__() must take 1 or 2 arguments. From %s'
+ msg = msg % (cls.__name__, inspect.getsourcefile(cls))
+ raise MachError(msg)
+
+ if len(spec.args) == 2:
+ pass_context = True
+
+ # We scan __dict__ because we only care about the classes own attributes,
+ # not inherited ones. If we did inherited attributes, we could potentially
+ # define commands multiple times. We also sort keys so commands defined in
+ # the same class are grouped in a sane order.
+ for attr in sorted(cls.__dict__.keys()):
+ value = cls.__dict__[attr]
+
+ if not isinstance(value, types.FunctionType):
+ continue
+
+ command_name, category, description, allow_all, conditions, parser = getattr(
+ value, '_mach_command', (None, None, None, None, None, None))
+
+ if command_name is None:
+ continue
+
+ if conditions is None and Registrar.require_conditions:
+ continue
+
+ msg = 'Mach command \'%s\' implemented incorrectly. ' + \
+ 'Conditions argument must take a list ' + \
+ 'of functions. Found %s instead.'
+
+ conditions = conditions or []
+ if not isinstance(conditions, collections.Iterable):
+ msg = msg % (command_name, type(conditions))
+ raise MachError(msg)
+
+ for c in conditions:
+ if not hasattr(c, '__call__'):
+ msg = msg % (command_name, type(c))
+ raise MachError(msg)
+
+ arguments = getattr(value, '_mach_command_args', None)
+
+ handler = MethodHandler(cls, attr, command_name, category=category,
+ description=description, allow_all_arguments=allow_all,
+ conditions=conditions, parser=parser, arguments=arguments,
+ pass_context=pass_context)
+
+ Registrar.register_command_handler(handler)
+
+ return cls
+
+
+class Command(object):
+ """Decorator for functions or methods that provide a mach subcommand.
+
+ The decorator accepts arguments that define basic attributes of the
+ command. The following arguments are recognized:
+
+ category -- The string category to which this command belongs. Mach's
+ help will group commands by category.
+
+ description -- A brief description of what the command does.
+
+ allow_all_args -- Bool indicating whether to allow unknown arguments
+ through to the command.
+
+ parser -- an optional argparse.ArgumentParser instance to use as
+ the basis for the command arguments.
+
+ For example:
+
+ @Command('foo', category='misc', description='Run the foo action')
+ def foo(self):
+ pass
+ """
+ def __init__(self, name, category=None, description=None,
+ allow_all_args=False, conditions=None, parser=None):
+ self._name = name
+ self._category = category
+ self._description = description
+ self._allow_all_args = allow_all_args
+ self._conditions = conditions
+ self._parser = parser
+
+ def __call__(self, func):
+ func._mach_command = (self._name, self._category, self._description,
+ self._allow_all_args, self._conditions, self._parser)
+
+ return func
+
+
+class CommandArgument(object):
+ """Decorator for additional arguments to mach subcommands.
+
+ This decorator should be used to add arguments to mach commands. Arguments
+ to the decorator are proxied to ArgumentParser.add_argument().
+
+ For example:
+
+ @Command('foo', help='Run the foo action')
+ @CommandArgument('-b', '--bar', action='store_true', default=False,
+ help='Enable bar mode.')
+ def foo(self):
+ pass
+ """
+ def __init__(self, *args, **kwargs):
+ self._command_args = (args, kwargs)
+
+ def __call__(self, func):
+ command_args = getattr(func, '_mach_command_args', [])
+
+ command_args.insert(0, self._command_args)
+
+ func._mach_command_args = command_args
+
+ return func
+
+
+def SettingsProvider(cls):
+ """Class decorator to denote that this class provides Mach settings.
+
+ When this decorator is encountered, the underlying class will automatically
+ be registered with the Mach registrar and will (likely) be hooked up to the
+ mach driver.
+
+ This decorator is only allowed on mach.config.ConfigProvider classes.
+ """
+ if not issubclass(cls, ConfigProvider):
+ raise MachError('@SettingsProvider encountered on class that does ' +
+ 'not derived from mach.config.ConfigProvider.')
+
+ Registrar.register_settings_provider(cls)
+
+ return cls
+
diff --git a/python/mach/mach/dispatcher.py b/python/mach/mach/dispatcher.py
new file mode 100644
index 00000000000..11e58cb9e2f
--- /dev/null
+++ b/python/mach/mach/dispatcher.py
@@ -0,0 +1,277 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import argparse
+import difflib
+import sys
+
+from operator import itemgetter
+
+from .base import (
+ NoCommandError,
+ UnknownCommandError,
+ UnrecognizedArgumentError,
+)
+
+
+class CommandFormatter(argparse.HelpFormatter):
+ """Custom formatter to format just a subcommand."""
+
+ def add_usage(self, *args):
+ pass
+
+
+class CommandAction(argparse.Action):
+ """An argparse action that handles mach commands.
+
+ This class is essentially a reimplementation of argparse's sub-parsers
+ feature. We first tried to use sub-parsers. However, they were missing
+ features like grouping of commands (http://bugs.python.org/issue14037).
+
+ The way this works involves light magic and a partial understanding of how
+ argparse works.
+
+ Arguments registered with an argparse.ArgumentParser have an action
+ associated with them. An action is essentially a class that when called
+ does something with the encountered argument(s). This class is one of those
+ action classes.
+
+ An instance of this class is created doing something like:
+
+ parser.add_argument('command', action=CommandAction, registrar=r)
+
+ Note that a mach.registrar.Registrar instance is passed in. The Registrar
+ holds information on all the mach commands that have been registered.
+
+ When this argument is registered with the ArgumentParser, an instance of
+ this class is instantiated. One of the subtle but important things it does
+ is tell the argument parser that it's interested in *all* of the remaining
+ program arguments. So, when the ArgumentParser calls this action, we will
+ receive the command name plus all of its arguments.
+
+ For more, read the docs in __call__.
+ """
+ def __init__(self, option_strings, dest, required=True, default=None,
+ registrar=None, context=None):
+ # A proper API would have **kwargs here. However, since we are a little
+ # hacky, we intentionally omit it as a way of detecting potentially
+ # breaking changes with argparse's implementation.
+ #
+ # In a similar vein, default is passed in but is not needed, so we drop
+ # it.
+ argparse.Action.__init__(self, option_strings, dest, required=required,
+ help=argparse.SUPPRESS, nargs=argparse.REMAINDER)
+
+ self._mach_registrar = registrar
+ self._context = context
+
+ def __call__(self, parser, namespace, values, option_string=None):
+ """This is called when the ArgumentParser has reached our arguments.
+
+ Since we always register ourselves with nargs=argparse.REMAINDER,
+ values should be a list of remaining arguments to parse. The first
+ argument should be the name of the command to invoke and all remaining
+ arguments are arguments for that command.
+
+ The gist of the flow is that we look at the command being invoked. If
+ it's *help*, we handle that specially (because argparse's default help
+ handler isn't satisfactory). Else, we create a new, independent
+ ArgumentParser instance for just the invoked command (based on the
+ information contained in the command registrar) and feed the arguments
+ into that parser. We then merge the results with the main
+ ArgumentParser.
+ """
+ if namespace.help:
+ # -h or --help is in the global arguments.
+ self._handle_main_help(parser, namespace.verbose)
+ sys.exit(0)
+ elif values:
+ command = values[0].lower()
+ args = values[1:]
+
+ if command == 'help':
+ if args and args[0] not in ['-h', '--help']:
+ # Make sure args[0] is indeed a command.
+ self._handle_subcommand_help(parser, args[0])
+ else:
+ self._handle_main_help(parser, namespace.verbose)
+ sys.exit(0)
+ elif '-h' in args or '--help' in args:
+ # -h or --help is in the command arguments.
+ self._handle_subcommand_help(parser, command)
+ sys.exit(0)
+ else:
+ raise NoCommandError()
+
+ # Command suggestion
+ if command not in self._mach_registrar.command_handlers:
+ # We first try to look for a valid command that is very similar to the given command.
+ suggested_commands = difflib.get_close_matches(command, self._mach_registrar.command_handlers.keys(), cutoff=0.8)
+ # If we find more than one matching command, or no command at all, we give command suggestions instead
+ # (with a lower matching threshold). All commands that start with the given command (for instance: 'mochitest-plain',
+ # 'mochitest-chrome', etc. for 'mochitest-') are also included.
+ if len(suggested_commands) != 1:
+ suggested_commands = set(difflib.get_close_matches(command, self._mach_registrar.command_handlers.keys(), cutoff=0.5))
+ suggested_commands |= {cmd for cmd in self._mach_registrar.command_handlers if cmd.startswith(command)}
+ raise UnknownCommandError(command, 'run', suggested_commands)
+ sys.stderr.write("We're assuming the '%s' command is '%s' and we're executing it for you.\n\n" % (command, suggested_commands[0]))
+ command = suggested_commands[0]
+
+ handler = self._mach_registrar.command_handlers.get(command)
+
+ # FUTURE
+ # If we wanted to conditionally enable commands based on whether
+ # it's possible to run them given the current state of system, here
+ # would be a good place to hook that up.
+
+ # We create a new parser, populate it with the command's arguments,
+ # then feed all remaining arguments to it, merging the results
+ # with ourselves. This is essentially what argparse subparsers
+ # do.
+
+ parser_args = {
+ 'add_help': False,
+ 'usage': '%(prog)s [global arguments] ' + command +
+ ' command arguments]',
+ }
+
+ if handler.allow_all_arguments:
+ parser_args['prefix_chars'] = '+'
+
+ if handler.parser:
+ subparser = handler.parser
+ else:
+ subparser = argparse.ArgumentParser(**parser_args)
+
+ for arg in handler.arguments:
+ subparser.add_argument(*arg[0], **arg[1])
+
+ # We define the command information on the main parser result so as to
+ # not interfere with arguments passed to the command.
+ setattr(namespace, 'mach_handler', handler)
+ setattr(namespace, 'command', command)
+
+ command_namespace, extra = subparser.parse_known_args(args)
+ setattr(namespace, 'command_args', command_namespace)
+ if extra:
+ raise UnrecognizedArgumentError(command, extra)
+
+ def _handle_main_help(self, parser, verbose):
+ # Since we don't need full sub-parser support for the main help output,
+ # we create groups in the ArgumentParser and populate each group with
+ # arguments corresponding to command names. This has the side-effect
+ # that argparse renders it nicely.
+ r = self._mach_registrar
+ disabled_commands = []
+
+ cats = [(k, v[2]) for k, v in r.categories.items()]
+ sorted_cats = sorted(cats, key=itemgetter(1), reverse=True)
+ for category, priority in sorted_cats:
+ group = None
+
+ for command in sorted(r.commands_by_category[category]):
+ handler = r.command_handlers[command]
+
+ # Instantiate a handler class to see if it should be filtered
+ # out for the current context or not. Condition functions can be
+ # applied to the command's decorator.
+ if handler.conditions:
+ if handler.pass_context:
+ instance = handler.cls(self._context)
+ else:
+ instance = handler.cls()
+
+ is_filtered = False
+ for c in handler.conditions:
+ if not c(instance):
+ is_filtered = True
+ break
+ if is_filtered:
+ description = handler.description
+ disabled_command = {'command': command, 'description': description}
+ disabled_commands.append(disabled_command)
+ continue
+
+ if group is None:
+ title, description, _priority = r.categories[category]
+ group = parser.add_argument_group(title, description)
+
+ description = handler.description
+ group.add_argument(command, help=description,
+ action='store_true')
+
+ if disabled_commands and 'disabled' in r.categories:
+ title, description, _priority = r.categories['disabled']
+ group = parser.add_argument_group(title, description)
+ if verbose == True:
+ for c in disabled_commands:
+ group.add_argument(c['command'], help=c['description'],
+ action='store_true')
+
+ parser.print_help()
+
+ def _handle_subcommand_help(self, parser, command):
+ handler = self._mach_registrar.command_handlers.get(command)
+
+ if not handler:
+ raise UnknownCommandError(command, 'query')
+
+ # This code is worth explaining. Because we are doing funky things with
+ # argument registration to allow the same option in both global and
+ # command arguments, we can't simply put all arguments on the same
+ # parser instance because argparse would complain. We can't register an
+ # argparse subparser here because it won't properly show help for
+ # global arguments. So, we employ a strategy similar to command
+ # execution where we construct a 2nd, independent ArgumentParser for
+ # just the command data then supplement the main help's output with
+ # this 2nd parser's. We use a custom formatter class to ignore some of
+ # the help output.
+ parser_args = {
+ 'formatter_class': CommandFormatter,
+ 'add_help': False,
+ }
+
+ if handler.allow_all_arguments:
+ parser_args['prefix_chars'] = '+'
+
+ if handler.parser:
+ c_parser = handler.parser
+ c_parser.formatter_class = NoUsageFormatter
+ # Accessing _action_groups is a bit shady. We are highly dependent
+ # on the argparse implementation not changing. We fail fast to
+ # detect upstream changes so we can intelligently react to them.
+ group = c_parser._action_groups[1]
+
+ # By default argparse adds two groups called "positional arguments"
+ # and "optional arguments". We want to rename these to reflect standard
+ # mach terminology.
+ c_parser._action_groups[0].title = 'Command Parameters'
+ c_parser._action_groups[1].title = 'Command Arguments'
+
+ if not handler.description:
+ handler.description = c_parser.description
+ c_parser.description = None
+ else:
+ c_parser = argparse.ArgumentParser(**parser_args)
+ group = c_parser.add_argument_group('Command Arguments')
+
+ for arg in handler.arguments:
+ group.add_argument(*arg[0], **arg[1])
+
+ # This will print the description of the command below the usage.
+ description = handler.description
+ if description:
+ parser.description = description
+
+ parser.usage = '%(prog)s [global arguments] ' + command + \
+ ' [command arguments]'
+ parser.print_help()
+ print('')
+ c_parser.print_help()
+
+class NoUsageFormatter(argparse.HelpFormatter):
+ def _format_usage(self, *args, **kwargs):
+ return ""
diff --git a/python/mach/mach/logging.py b/python/mach/mach/logging.py
new file mode 100644
index 00000000000..729e6cb3d93
--- /dev/null
+++ b/python/mach/mach/logging.py
@@ -0,0 +1,256 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This file contains logging functionality for mach. It essentially provides
+# support for a structured logging framework built on top of Python's built-in
+# logging framework.
+
+from __future__ import absolute_import, unicode_literals
+
+try:
+ import blessings
+except ImportError:
+ blessings = None
+
+import json
+import logging
+import sys
+import time
+
+
+def format_seconds(total):
+ """Format number of seconds to MM:SS.DD form."""
+
+ minutes, seconds = divmod(total, 60)
+
+ return '%2d:%05.2f' % (minutes, seconds)
+
+
+class ConvertToStructuredFilter(logging.Filter):
+ """Filter that converts unstructured records into structured ones."""
+ def filter(self, record):
+ if hasattr(record, 'action') and hasattr(record, 'params'):
+ return True
+
+ record.action = 'unstructured'
+ record.params = {'msg': record.getMessage()}
+ record.msg = '{msg}'
+
+ return True
+
+
+class StructuredJSONFormatter(logging.Formatter):
+ """Log formatter that writes a structured JSON entry."""
+
+ def format(self, record):
+ action = getattr(record, 'action', 'UNKNOWN')
+ params = getattr(record, 'params', {})
+
+ return json.dumps([record.created, action, params])
+
+
+class StructuredHumanFormatter(logging.Formatter):
+ """Log formatter that writes structured messages for humans.
+
+ It is important that this formatter never be added to a logger that
+ produces unstructured/classic log messages. If it is, the call to format()
+ could fail because the string could contain things (like JSON) that look
+ like formatting character sequences.
+
+ Because of this limitation, format() will fail with a KeyError if an
+ unstructured record is passed or if the structured message is malformed.
+ """
+ def __init__(self, start_time, write_interval=False, write_times=True):
+ self.start_time = start_time
+ self.write_interval = write_interval
+ self.write_times = write_times
+ self.last_time = None
+
+ def format(self, record):
+ f = record.msg.format(**record.params)
+
+ if not self.write_times:
+ return f
+
+ elapsed = self._time(record)
+
+ return '%s %s' % (format_seconds(elapsed), f)
+
+ def _time(self, record):
+ t = record.created - self.start_time
+
+ if self.write_interval and self.last_time is not None:
+ t = record.created - self.last_time
+
+ self.last_time = record.created
+
+ return t
+
+
+class StructuredTerminalFormatter(StructuredHumanFormatter):
+ """Log formatter for structured messages writing to a terminal."""
+
+ def set_terminal(self, terminal):
+ self.terminal = terminal
+
+ def format(self, record):
+ f = record.msg.format(**record.params)
+
+ if not self.write_times:
+ return f
+
+ t = self.terminal.blue(format_seconds(self._time(record)))
+
+ return '%s %s' % (t, self._colorize(f))
+
+ def _colorize(self, s):
+ if not self.terminal:
+ return s
+
+ result = s
+
+ reftest = s.startswith('REFTEST ')
+ if reftest:
+ s = s[8:]
+
+ if s.startswith('TEST-PASS'):
+ result = self.terminal.green(s[0:9]) + s[9:]
+ elif s.startswith('TEST-UNEXPECTED'):
+ result = self.terminal.red(s[0:20]) + s[20:]
+ elif s.startswith('TEST-START'):
+ result = self.terminal.yellow(s[0:10]) + s[10:]
+ elif s.startswith('TEST-INFO'):
+ result = self.terminal.yellow(s[0:9]) + s[9:]
+
+ if reftest:
+ result = 'REFTEST ' + result
+
+ return result
+
+
+class LoggingManager(object):
+ """Holds and controls global logging state.
+
+ An application should instantiate one of these and configure it as needed.
+
+ This class provides a mechanism to configure the output of logging data
+ both from mach and from the overall logging system (e.g. from other
+ modules).
+ """
+
+ def __init__(self):
+ self.start_time = time.time()
+
+ self.json_handlers = []
+ self.terminal_handler = None
+ self.terminal_formatter = None
+
+ self.root_logger = logging.getLogger()
+ self.root_logger.setLevel(logging.DEBUG)
+
+ # Installing NullHandler on the root logger ensures that *all* log
+ # messages have at least one handler. This prevents Python from
+ # complaining about "no handlers could be found for logger XXX."
+ self.root_logger.addHandler(logging.NullHandler())
+
+ self.mach_logger = logging.getLogger('mach')
+ self.mach_logger.setLevel(logging.DEBUG)
+
+ self.structured_filter = ConvertToStructuredFilter()
+
+ self.structured_loggers = [self.mach_logger]
+
+ self._terminal = None
+
+ @property
+ def terminal(self):
+ if not self._terminal and blessings:
+ # Sometimes blessings fails to set up the terminal. In that case,
+ # silently fail.
+ try:
+ terminal = blessings.Terminal(stream=sys.stdout)
+
+ if terminal.is_a_tty:
+ self._terminal = terminal
+ except Exception:
+ pass
+
+ return self._terminal
+
+ def add_json_handler(self, fh):
+ """Enable JSON logging on the specified file object."""
+
+ # Configure the consumer of structured messages.
+ handler = logging.StreamHandler(stream=fh)
+ handler.setFormatter(StructuredJSONFormatter())
+ handler.setLevel(logging.DEBUG)
+
+ # And hook it up.
+ for logger in self.structured_loggers:
+ logger.addHandler(handler)
+
+ self.json_handlers.append(handler)
+
+ def add_terminal_logging(self, fh=sys.stdout, level=logging.INFO,
+ write_interval=False, write_times=True):
+ """Enable logging to the terminal."""
+
+ formatter = StructuredHumanFormatter(self.start_time,
+ write_interval=write_interval, write_times=write_times)
+
+ if self.terminal:
+ formatter = StructuredTerminalFormatter(self.start_time,
+ write_interval=write_interval, write_times=write_times)
+ formatter.set_terminal(self.terminal)
+
+ handler = logging.StreamHandler(stream=fh)
+ handler.setFormatter(formatter)
+ handler.setLevel(level)
+
+ for logger in self.structured_loggers:
+ logger.addHandler(handler)
+
+ self.terminal_handler = handler
+ self.terminal_formatter = formatter
+
+ def replace_terminal_handler(self, handler):
+ """Replace the installed terminal handler.
+
+ Returns the old handler or None if none was configured.
+ If the new handler is None, removes any existing handler and disables
+ logging to the terminal.
+ """
+ old = self.terminal_handler
+
+ if old:
+ for logger in self.structured_loggers:
+ logger.removeHandler(old)
+
+ if handler:
+ for logger in self.structured_loggers:
+ logger.addHandler(handler)
+
+ self.terminal_handler = handler
+
+ return old
+
+ def enable_unstructured(self):
+ """Enable logging of unstructured messages."""
+ if self.terminal_handler:
+ self.terminal_handler.addFilter(self.structured_filter)
+ self.root_logger.addHandler(self.terminal_handler)
+
+ def disable_unstructured(self):
+ """Disable logging of unstructured messages."""
+ if self.terminal_handler:
+ self.terminal_handler.removeFilter(self.structured_filter)
+ self.root_logger.removeHandler(self.terminal_handler)
+
+ def register_structured_logger(self, logger):
+ """Register a structured logger.
+
+ This needs to be called for all structured loggers that don't chain up
+ to the mach logger in order for their output to be captured.
+ """
+ self.structured_loggers.append(logger)
diff --git a/python/mach/mach/main.py b/python/mach/mach/main.py
new file mode 100644
index 00000000000..3c74bd51333
--- /dev/null
+++ b/python/mach/mach/main.py
@@ -0,0 +1,615 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This module provides functionality for the command-line build tool
+# (mach). It is packaged as a module because everything is a library.
+
+from __future__ import absolute_import, print_function, unicode_literals
+from collections import Iterable
+
+import argparse
+import codecs
+import imp
+import logging
+import os
+import sys
+import traceback
+import uuid
+import sys
+
+from .base import (
+ CommandContext,
+ MachError,
+ NoCommandError,
+ UnknownCommandError,
+ UnrecognizedArgumentError,
+)
+
+from .decorators import (
+ CommandArgument,
+ CommandProvider,
+ Command,
+)
+
+from .config import ConfigSettings
+from .dispatcher import CommandAction
+from .logging import LoggingManager
+from .registrar import Registrar
+
+
+
+MACH_ERROR = r'''
+The error occurred in mach itself. This is likely a bug in mach itself or a
+fundamental problem with a loaded module.
+
+Please consider filing a bug against mach by going to the URL:
+
+ https://bugzilla.mozilla.org/enter_bug.cgi?product=Core&component=mach
+
+'''.lstrip()
+
+ERROR_FOOTER = r'''
+If filing a bug, please include the full output of mach, including this error
+message.
+
+The details of the failure are as follows:
+'''.lstrip()
+
+COMMAND_ERROR = r'''
+The error occurred in the implementation of the invoked mach command.
+
+This should never occur and is likely a bug in the implementation of that
+command. Consider filing a bug for this issue.
+'''.lstrip()
+
+MODULE_ERROR = r'''
+The error occurred in code that was called by the mach command. This is either
+a bug in the called code itself or in the way that mach is calling it.
+
+You should consider filing a bug for this issue.
+'''.lstrip()
+
+NO_COMMAND_ERROR = r'''
+It looks like you tried to run mach without a command.
+
+Run |mach help| to show a list of commands.
+'''.lstrip()
+
+UNKNOWN_COMMAND_ERROR = r'''
+It looks like you are trying to %s an unknown mach command: %s
+%s
+Run |mach help| to show a list of commands.
+'''.lstrip()
+
+SUGGESTED_COMMANDS_MESSAGE = r'''
+Did you want to %s any of these commands instead: %s?
+'''
+
+UNRECOGNIZED_ARGUMENT_ERROR = r'''
+It looks like you passed an unrecognized argument into mach.
+
+The %s command does not accept the arguments: %s
+'''.lstrip()
+
+INVALID_COMMAND_CONTEXT = r'''
+It looks like you tried to run a mach command from an invalid context. The %s
+command failed to meet the following conditions: %s
+
+Run |mach help| to show a list of all commands available to the current context.
+'''.lstrip()
+
+INVALID_ENTRY_POINT = r'''
+Entry points should return a list of command providers or directories
+containing command providers. The following entry point is invalid:
+
+ %s
+
+You are seeing this because there is an error in an external module attempting
+to implement a mach command. Please fix the error, or uninstall the module from
+your system.
+'''.lstrip()
+
+class ArgumentParser(argparse.ArgumentParser):
+ """Custom implementation argument parser to make things look pretty."""
+
+ def error(self, message):
+ """Custom error reporter to give more helpful text on bad commands."""
+ if not message.startswith('argument command: invalid choice'):
+ argparse.ArgumentParser.error(self, message)
+ assert False
+
+ print('Invalid command specified. The list of commands is below.\n')
+ self.print_help()
+ sys.exit(1)
+
+ def format_help(self):
+ text = argparse.ArgumentParser.format_help(self)
+
+ # Strip out the silly command list that would preceed the pretty list.
+ #
+ # Commands:
+ # {foo,bar}
+ # foo Do foo.
+ # bar Do bar.
+ search = 'Commands:\n {'
+ start = text.find(search)
+
+ if start != -1:
+ end = text.find('}\n', start)
+ assert end != -1
+
+ real_start = start + len('Commands:\n')
+ real_end = end + len('}\n')
+
+ text = text[0:real_start] + text[real_end:]
+
+ return text
+
+
+class ContextWrapper(object):
+ def __init__(self, context, handler):
+ object.__setattr__(self, '_context', context)
+ object.__setattr__(self, '_handler', handler)
+
+ def __getattribute__(self, key):
+ try:
+ return getattr(object.__getattribute__(self, '_context'), key)
+ except AttributeError as e:
+ try:
+ ret = object.__getattribute__(self, '_handler')(self, key)
+ except AttributeError, TypeError:
+ # TypeError is in case the handler comes from old code not
+ # taking a key argument.
+ raise e
+ setattr(self, key, ret)
+ return ret
+
+ def __setattr__(self, key, value):
+ setattr(object.__getattribute__(self, '_context'), key, value)
+
+
+@CommandProvider
+class Mach(object):
+ """Main mach driver type.
+
+ This type is responsible for holding global mach state and dispatching
+ a command from arguments.
+
+ The following attributes may be assigned to the instance to influence
+ behavior:
+
+ populate_context_handler -- If defined, it must be a callable. The
+ callable signature is the following:
+ populate_context_handler(context, key=None)
+ It acts as a fallback getter for the mach.base.CommandContext
+ instance.
+ This allows to augment the context instance with arbitrary data
+ for use in command handlers.
+ For backwards compatibility, it is also called before command
+ dispatch without a key, allowing the context handler to add
+ attributes to the context instance.
+
+ require_conditions -- If True, commands that do not have any condition
+ functions applied will be skipped. Defaults to False.
+
+ """
+
+ USAGE = """%(prog)s [global arguments] command [command arguments]
+
+mach (German for "do") is the main interface to the Mozilla build system and
+common developer tasks.
+
+You tell mach the command you want to perform and it does it for you.
+
+Some common commands are:
+
+ %(prog)s build Build/compile the source tree.
+ %(prog)s help Show full help, including the list of all commands.
+
+To see more help for a specific command, run:
+
+ %(prog)s help <command>
+"""
+
+ def __init__(self, cwd):
+ assert os.path.isdir(cwd)
+
+ self.cwd = cwd
+ self.log_manager = LoggingManager()
+ self.logger = logging.getLogger(__name__)
+ self.settings = ConfigSettings()
+
+ self.log_manager.register_structured_logger(self.logger)
+ self.global_arguments = []
+ self.populate_context_handler = None
+
+ def add_global_argument(self, *args, **kwargs):
+ """Register a global argument with the argument parser.
+
+ Arguments are proxied to ArgumentParser.add_argument()
+ """
+
+ self.global_arguments.append((args, kwargs))
+
+ def load_commands_from_directory(self, path):
+ """Scan for mach commands from modules in a directory.
+
+ This takes a path to a directory, loads the .py files in it, and
+ registers and found mach command providers with this mach instance.
+ """
+ for f in sorted(os.listdir(path)):
+ if not f.endswith('.py') or f == '__init__.py':
+ continue
+
+ full_path = os.path.join(path, f)
+ module_name = 'mach.commands.%s' % f[0:-3]
+
+ self.load_commands_from_file(full_path, module_name=module_name)
+
+ def load_commands_from_file(self, path, module_name=None):
+ """Scan for mach commands from a file.
+
+ This takes a path to a file and loads it as a Python module under the
+ module name specified. If no name is specified, a random one will be
+ chosen.
+ """
+ if module_name is None:
+ # Ensure parent module is present otherwise we'll (likely) get
+ # an error due to unknown parent.
+ if b'mach.commands' not in sys.modules:
+ mod = imp.new_module(b'mach.commands')
+ sys.modules[b'mach.commands'] = mod
+
+ module_name = 'mach.commands.%s' % uuid.uuid1().get_hex()
+
+ imp.load_source(module_name, path)
+
+ def load_commands_from_entry_point(self, group='mach.providers'):
+ """Scan installed packages for mach command provider entry points. An
+ entry point is a function that returns a list of paths to files or
+ directories containing command providers.
+
+ This takes an optional group argument which specifies the entry point
+ group to use. If not specified, it defaults to 'mach.providers'.
+ """
+ try:
+ import pkg_resources
+ except ImportError:
+ print("Could not find setuptools, ignoring command entry points",
+ file=sys.stderr)
+ return
+
+ for entry in pkg_resources.iter_entry_points(group=group, name=None):
+ paths = entry.load()()
+ if not isinstance(paths, Iterable):
+ print(INVALID_ENTRY_POINT % entry)
+ sys.exit(1)
+
+ for path in paths:
+ if os.path.isfile(path):
+ self.load_commands_from_file(path)
+ elif os.path.isdir(path):
+ self.load_commands_from_directory(path)
+ else:
+ print("command provider '%s' does not exist" % path)
+
+ def define_category(self, name, title, description, priority=50):
+ """Provide a description for a named command category."""
+
+ Registrar.register_category(name, title, description, priority)
+
+ @property
+ def require_conditions(self):
+ return Registrar.require_conditions
+
+ @require_conditions.setter
+ def require_conditions(self, value):
+ Registrar.require_conditions = value
+
+ def run(self, argv, stdin=None, stdout=None, stderr=None):
+ """Runs mach with arguments provided from the command line.
+
+ Returns the integer exit code that should be used. 0 means success. All
+ other values indicate failure.
+ """
+
+ # If no encoding is defined, we default to UTF-8 because without this
+ # Python 2.7 will assume the default encoding of ASCII. This will blow
+ # up with UnicodeEncodeError as soon as it encounters a non-ASCII
+ # character in a unicode instance. We simply install a wrapper around
+ # the streams and restore once we have finished.
+ stdin = sys.stdin if stdin is None else stdin
+ stdout = sys.stdout if stdout is None else stdout
+ stderr = sys.stderr if stderr is None else stderr
+
+ orig_stdin = sys.stdin
+ orig_stdout = sys.stdout
+ orig_stderr = sys.stderr
+
+ sys.stdin = stdin
+ sys.stdout = stdout
+ sys.stderr = stderr
+
+ try:
+ if stdin.encoding is None:
+ sys.stdin = codecs.getreader('utf-8')(stdin)
+
+ if stdout.encoding is None:
+ sys.stdout = codecs.getwriter('utf-8')(stdout)
+
+ if stderr.encoding is None:
+ sys.stderr = codecs.getwriter('utf-8')(stderr)
+
+ return self._run(argv)
+ except KeyboardInterrupt:
+ print('mach interrupted by signal or user action. Stopping.')
+ return 1
+
+ except Exception as e:
+ # _run swallows exceptions in invoked handlers and converts them to
+ # a proper exit code. So, the only scenario where we should get an
+ # exception here is if _run itself raises. If _run raises, that's a
+ # bug in mach (or a loaded command module being silly) and thus
+ # should be reported differently.
+ self._print_error_header(argv, sys.stdout)
+ print(MACH_ERROR)
+
+ exc_type, exc_value, exc_tb = sys.exc_info()
+ stack = traceback.extract_tb(exc_tb)
+
+ self._print_exception(sys.stdout, exc_type, exc_value, stack)
+
+ return 1
+
+ finally:
+ sys.stdin = orig_stdin
+ sys.stdout = orig_stdout
+ sys.stderr = orig_stderr
+
+ def _run(self, argv):
+ context = CommandContext(cwd=self.cwd,
+ settings=self.settings, log_manager=self.log_manager,
+ commands=Registrar)
+
+ if self.populate_context_handler:
+ self.populate_context_handler(context)
+ context = ContextWrapper(context, self.populate_context_handler)
+
+ parser = self.get_argument_parser(context)
+
+ if not len(argv):
+ # We don't register the usage until here because if it is globally
+ # registered, argparse always prints it. This is not desired when
+ # running with --help.
+ parser.usage = Mach.USAGE
+ parser.print_usage()
+ return 0
+
+ try:
+ args = parser.parse_args(argv)
+ except NoCommandError:
+ print(NO_COMMAND_ERROR)
+ return 1
+ except UnknownCommandError as e:
+ suggestion_message = SUGGESTED_COMMANDS_MESSAGE % (e.verb, ', '.join(e.suggested_commands)) if e.suggested_commands else ''
+ print(UNKNOWN_COMMAND_ERROR % (e.verb, e.command, suggestion_message))
+ return 1
+ except UnrecognizedArgumentError as e:
+ print(UNRECOGNIZED_ARGUMENT_ERROR % (e.command,
+ ' '.join(e.arguments)))
+ return 1
+
+ # Add JSON logging to a file if requested.
+ if args.logfile:
+ self.log_manager.add_json_handler(args.logfile)
+
+ # Up the logging level if requested.
+ log_level = logging.INFO
+ if args.verbose:
+ log_level = logging.DEBUG
+
+ self.log_manager.register_structured_logger(logging.getLogger('mach'))
+
+ write_times = True
+ if args.log_no_times or 'MACH_NO_WRITE_TIMES' in os.environ:
+ write_times = False
+
+ # Always enable terminal logging. The log manager figures out if we are
+ # actually in a TTY or are a pipe and does the right thing.
+ self.log_manager.add_terminal_logging(level=log_level,
+ write_interval=args.log_interval, write_times=write_times)
+
+ self.load_settings(args)
+
+ if not hasattr(args, 'mach_handler'):
+ raise MachError('ArgumentParser result missing mach handler info.')
+
+ handler = getattr(args, 'mach_handler')
+ cls = handler.cls
+
+ if handler.pass_context:
+ instance = cls(context)
+ else:
+ instance = cls()
+
+ if handler.conditions:
+ fail_conditions = []
+ for c in handler.conditions:
+ if not c(instance):
+ fail_conditions.append(c)
+
+ if fail_conditions:
+ print(self._condition_failed_message(handler.name, fail_conditions))
+ return 1
+
+ fn = getattr(instance, handler.method)
+
+ try:
+ result = fn(**vars(args.command_args))
+
+ if not result:
+ result = 0
+
+ assert isinstance(result, (int, long))
+
+ return result
+ except KeyboardInterrupt as ki:
+ raise ki
+ except Exception as e:
+ exc_type, exc_value, exc_tb = sys.exc_info()
+
+ # The first frame is us and is never used.
+ stack = traceback.extract_tb(exc_tb)[1:]
+
+ # If we have nothing on the stack, the exception was raised as part
+ # of calling the @Command method itself. This likely means a
+ # mismatch between @CommandArgument and arguments to the method.
+ # e.g. there exists a @CommandArgument without the corresponding
+ # argument on the method. We handle that here until the module
+ # loader grows the ability to validate better.
+ if not len(stack):
+ print(COMMAND_ERROR)
+ self._print_exception(sys.stdout, exc_type, exc_value,
+ traceback.extract_tb(exc_tb))
+ return 1
+
+ # Split the frames into those from the module containing the
+ # command and everything else.
+ command_frames = []
+ other_frames = []
+
+ initial_file = stack[0][0]
+
+ for frame in stack:
+ if frame[0] == initial_file:
+ command_frames.append(frame)
+ else:
+ other_frames.append(frame)
+
+ # If the exception was in the module providing the command, it's
+ # likely the bug is in the mach command module, not something else.
+ # If there are other frames, the bug is likely not the mach
+ # command's fault.
+ self._print_error_header(argv, sys.stdout)
+
+ if len(other_frames):
+ print(MODULE_ERROR)
+ else:
+ print(COMMAND_ERROR)
+
+ self._print_exception(sys.stdout, exc_type, exc_value, stack)
+
+ return 1
+
+ def log(self, level, action, params, format_str):
+ """Helper method to record a structured log event."""
+ self.logger.log(level, format_str,
+ extra={'action': action, 'params': params})
+
+ @classmethod
+ def _condition_failed_message(cls, name, conditions):
+ msg = ['\n']
+ for c in conditions:
+ part = [' %s' % c.__name__]
+ if c.__doc__ is not None:
+ part.append(c.__doc__)
+ msg.append(' - '.join(part))
+ return INVALID_COMMAND_CONTEXT % (name, '\n'.join(msg))
+
+ def _print_error_header(self, argv, fh):
+ fh.write('Error running mach:\n\n')
+ fh.write(' ')
+ fh.write(repr(argv))
+ fh.write('\n\n')
+
+ def _print_exception(self, fh, exc_type, exc_value, stack):
+ fh.write(ERROR_FOOTER)
+ fh.write('\n')
+
+ for l in traceback.format_exception_only(exc_type, exc_value):
+ fh.write(l)
+
+ fh.write('\n')
+ for l in traceback.format_list(stack):
+ fh.write(l)
+
+ def load_settings(self, args):
+ """Determine which settings files apply and load them.
+
+ Currently, we only support loading settings from a single file.
+ Ideally, we support loading from multiple files. This is supported by
+ the ConfigSettings API. However, that API currently doesn't track where
+ individual values come from, so if we load from multiple sources then
+ save, we effectively do a full copy. We don't want this. Until
+ ConfigSettings does the right thing, we shouldn't expose multi-file
+ loading.
+
+ We look for a settings file in the following locations. The first one
+ found wins:
+
+ 1) Command line argument
+ 2) Environment variable
+ 3) Default path
+ """
+ # Settings are disabled until integration with command providers is
+ # worked out.
+ self.settings = None
+ return False
+
+ for provider in Registrar.settings_providers:
+ provider.register_settings()
+ self.settings.register_provider(provider)
+
+ p = os.path.join(self.cwd, 'mach.ini')
+
+ if args.settings_file:
+ p = args.settings_file
+ elif 'MACH_SETTINGS_FILE' in os.environ:
+ p = os.environ['MACH_SETTINGS_FILE']
+
+ self.settings.load_file(p)
+
+ return os.path.exists(p)
+
+ def get_argument_parser(self, context):
+ """Returns an argument parser for the command-line interface."""
+
+ parser = ArgumentParser(add_help=False,
+ usage='%(prog)s [global arguments] command [command arguments]')
+
+ # Order is important here as it dictates the order the auto-generated
+ # help messages are printed.
+ global_group = parser.add_argument_group('Global Arguments')
+
+ #global_group.add_argument('--settings', dest='settings_file',
+ # metavar='FILENAME', help='Path to settings file.')
+
+ global_group.add_argument('-v', '--verbose', dest='verbose',
+ action='store_true', default=False,
+ help='Print verbose output.')
+ global_group.add_argument('-l', '--log-file', dest='logfile',
+ metavar='FILENAME', type=argparse.FileType('ab'),
+ help='Filename to write log data to.')
+ global_group.add_argument('--log-interval', dest='log_interval',
+ action='store_true', default=False,
+ help='Prefix log line with interval from last message rather '
+ 'than relative time. Note that this is NOT execution time '
+ 'if there are parallel operations.')
+ global_group.add_argument('--log-no-times', dest='log_no_times',
+ action='store_true', default=False,
+ help='Do not prefix log lines with times. By default, mach will '
+ 'prefix each output line with the time since command start.')
+ global_group.add_argument('-h', '--help', dest='help',
+ action='store_true', default=False,
+ help='Show this help message.')
+
+ for args, kwargs in self.global_arguments:
+ global_group.add_argument(*args, **kwargs)
+
+ # We need to be last because CommandAction swallows all remaining
+ # arguments and argparse parses arguments in the order they were added.
+ parser.add_argument('command', action=CommandAction,
+ registrar=Registrar, context=context)
+
+ return parser
diff --git a/python/mach/mach/mixin/__init__.py b/python/mach/mach/mixin/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/python/mach/mach/mixin/__init__.py
diff --git a/python/mach/mach/mixin/logging.py b/python/mach/mach/mixin/logging.py
new file mode 100644
index 00000000000..5c37b54f1bd
--- /dev/null
+++ b/python/mach/mach/mixin/logging.py
@@ -0,0 +1,55 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, unicode_literals
+
+import logging
+
+
+class LoggingMixin(object):
+ """Provides functionality to control logging."""
+
+ def populate_logger(self, name=None):
+ """Ensure this class instance has a logger associated with it.
+
+ Users of this mixin that call log() will need to ensure self._logger is
+ a logging.Logger instance before they call log(). This function ensures
+ self._logger is defined by populating it if it isn't.
+ """
+ if hasattr(self, '_logger'):
+ return
+
+ if name is None:
+ name = '.'.join([self.__module__, self.__class__.__name__])
+
+ self._logger = logging.getLogger(name)
+
+ def log(self, level, action, params, format_str):
+ """Log a structured log event.
+
+ A structured log event consists of a logging level, a string action, a
+ dictionary of attributes, and a formatting string.
+
+ The logging level is one of the logging.* constants, such as
+ logging.INFO.
+
+ The action string is essentially the enumeration of the event. Each
+ different type of logged event should have a different action.
+
+ The params dict is the metadata constituting the logged event.
+
+ The formatting string is used to convert the structured message back to
+ human-readable format. Conversion back to human-readable form is
+ performed by calling format() on this string, feeding into it the dict
+ of attributes constituting the event.
+
+ Example Usage
+ -------------
+
+ self.log(logging.DEBUG, 'login', {'username': 'johndoe'},
+ 'User login: {username}')
+ """
+ self._logger.log(level, format_str,
+ extra={'action': action, 'params': params})
+
diff --git a/python/mach/mach/mixin/process.py b/python/mach/mach/mixin/process.py
new file mode 100644
index 00000000000..5b06da3b60e
--- /dev/null
+++ b/python/mach/mach/mixin/process.py
@@ -0,0 +1,175 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This module provides mixins to perform process execution.
+
+from __future__ import absolute_import, unicode_literals
+
+import logging
+import os
+import subprocess
+import sys
+
+from mozprocess.processhandler import ProcessHandlerMixin
+
+from .logging import LoggingMixin
+
+
+# Perform detection of operating system environment. This is used by command
+# execution. We only do this once to save redundancy. Yes, this can fail module
+# loading. That is arguably OK.
+if 'SHELL' in os.environ:
+ _current_shell = os.environ['SHELL']
+elif 'MOZILLABUILD' in os.environ:
+ _current_shell = os.environ['MOZILLABUILD'] + '/msys/bin/sh.exe'
+elif 'COMSPEC' in os.environ:
+ _current_shell = os.environ['COMSPEC']
+else:
+ raise Exception('Could not detect environment shell!')
+
+_in_msys = False
+
+if os.environ.get('MSYSTEM', None) == 'MINGW32':
+ _in_msys = True
+
+ if not _current_shell.lower().endswith('.exe'):
+ _current_shell += '.exe'
+
+
+class ProcessExecutionMixin(LoggingMixin):
+ """Mix-in that provides process execution functionality."""
+
+ def run_process(self, args=None, cwd=None, append_env=None,
+ explicit_env=None, log_name=None, log_level=logging.INFO,
+ line_handler=None, require_unix_environment=False,
+ ensure_exit_code=0, ignore_children=False, pass_thru=False):
+ """Runs a single process to completion.
+
+ Takes a list of arguments to run where the first item is the
+ executable. Runs the command in the specified directory and
+ with optional environment variables.
+
+ append_env -- Dict of environment variables to append to the current
+ set of environment variables.
+ explicit_env -- Dict of environment variables to set for the new
+ process. Any existing environment variables will be ignored.
+
+ require_unix_environment if True will ensure the command is executed
+ within a UNIX environment. Basically, if we are on Windows, it will
+ execute the command via an appropriate UNIX-like shell.
+
+ ignore_children is proxied to mozprocess's ignore_children.
+
+ ensure_exit_code is used to ensure the exit code of a process matches
+ what is expected. If it is an integer, we raise an Exception if the
+ exit code does not match this value. If it is True, we ensure the exit
+ code is 0. If it is False, we don't perform any exit code validation.
+
+ pass_thru is a special execution mode where the child process inherits
+ this process's standard file handles (stdin, stdout, stderr) as well as
+ additional file descriptors. It should be used for interactive processes
+ where buffering from mozprocess could be an issue. pass_thru does not
+ use mozprocess. Therefore, arguments like log_name, line_handler,
+ and ignore_children have no effect.
+ """
+ args = self._normalize_command(args, require_unix_environment)
+
+ self.log(logging.INFO, 'new_process', {'args': args}, ' '.join(args))
+
+ def handleLine(line):
+ # Converts str to unicode on Python 2 and bytes to str on Python 3.
+ if isinstance(line, bytes):
+ line = line.decode(sys.stdout.encoding or 'utf-8', 'replace')
+
+ if line_handler:
+ line_handler(line)
+
+ if not log_name:
+ return
+
+ self.log(log_level, log_name, {'line': line.rstrip()}, '{line}')
+
+ use_env = {}
+ if explicit_env:
+ use_env = explicit_env
+ else:
+ use_env.update(os.environ)
+
+ if append_env:
+ use_env.update(append_env)
+
+ self.log(logging.DEBUG, 'process', {'env': use_env}, 'Environment: {env}')
+
+ # There is a bug in subprocess where it doesn't like unicode types in
+ # environment variables. Here, ensure all unicode are converted to
+ # binary. utf-8 is our globally assumed default. If the caller doesn't
+ # want UTF-8, they shouldn't pass in a unicode instance.
+ normalized_env = {}
+ for k, v in use_env.items():
+ if isinstance(k, unicode):
+ k = k.encode('utf-8', 'strict')
+
+ if isinstance(v, unicode):
+ v = v.encode('utf-8', 'strict')
+
+ normalized_env[k] = v
+
+ use_env = normalized_env
+
+ if pass_thru:
+ proc = subprocess.Popen(args, cwd=cwd, env=use_env)
+ status = None
+ # Leave it to the subprocess to handle Ctrl+C. If it terminates as
+ # a result of Ctrl+C, proc.wait() will return a status code, and,
+ # we get out of the loop. If it doesn't, like e.g. gdb, we continue
+ # waiting.
+ while status is None:
+ try:
+ status = proc.wait()
+ except KeyboardInterrupt:
+ pass
+ else:
+ p = ProcessHandlerMixin(args, cwd=cwd, env=use_env,
+ processOutputLine=[handleLine], universal_newlines=True,
+ ignore_children=ignore_children)
+ p.run()
+ p.processOutput()
+ status = p.wait()
+
+ if ensure_exit_code is False:
+ return status
+
+ if ensure_exit_code is True:
+ ensure_exit_code = 0
+
+ if status != ensure_exit_code:
+ raise Exception('Process executed with non-0 exit code: %s' % args)
+
+ return status
+
+ def _normalize_command(self, args, require_unix_environment):
+ """Adjust command arguments to run in the necessary environment.
+
+ This exists mainly to facilitate execution of programs requiring a *NIX
+ shell when running on Windows. The caller specifies whether a shell
+ environment is required. If it is and we are running on Windows but
+ aren't running in the UNIX-like msys environment, then we rewrite the
+ command to execute via a shell.
+ """
+ assert isinstance(args, list) and len(args)
+
+ if not require_unix_environment or not _in_msys:
+ return args
+
+ # Always munge Windows-style into Unix style for the command.
+ prog = args[0].replace('\\', '/')
+
+ # PyMake removes the C: prefix. But, things seem to work here
+ # without it. Not sure what that's about.
+
+ # We run everything through the msys shell. We need to use
+ # '-c' and pass all the arguments as one argument because that is
+ # how sh works.
+ cline = subprocess.list2cmdline([prog] + args[1:])
+ return [_current_shell, '-c', cline]
diff --git a/python/mach/mach/registrar.py b/python/mach/mach/registrar.py
new file mode 100644
index 00000000000..a2de24b2d04
--- /dev/null
+++ b/python/mach/mach/registrar.py
@@ -0,0 +1,65 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+from .base import MachError
+
+
+class MachRegistrar(object):
+ """Container for mach command and config providers."""
+
+ def __init__(self):
+ self.command_handlers = {}
+ self.commands_by_category = {}
+ self.settings_providers = set()
+ self.categories = {}
+ self.require_conditions = False
+
+ def register_command_handler(self, handler):
+ name = handler.name
+
+ if not handler.category:
+ raise MachError('Cannot register a mach command without a '
+ 'category: %s' % name)
+
+ if handler.category not in self.categories:
+ raise MachError('Cannot register a command to an undefined '
+ 'category: %s -> %s' % (name, handler.category))
+
+ self.command_handlers[name] = handler
+ self.commands_by_category[handler.category].add(name)
+
+ def register_settings_provider(self, cls):
+ self.settings_providers.add(cls)
+
+ def register_category(self, name, title, description, priority=50):
+ self.categories[name] = (title, description, priority)
+ self.commands_by_category[name] = set()
+
+ def dispatch(self, name, context=None, **args):
+ """Dispatch/run a command.
+
+ Commands can use this to call other commands.
+ """
+
+ # TODO The logic in this function overlaps with code in
+ # mach.main.Main._run() and should be consolidated.
+ handler = self.command_handlers[name]
+ cls = handler.cls
+
+ if handler.pass_context and not context:
+ raise Exception('mach command class requires context.')
+
+ if handler.pass_context:
+ instance = cls(context)
+ else:
+ instance = cls()
+
+ fn = getattr(instance, handler.method)
+
+ return fn(**args) or 0
+
+
+Registrar = MachRegistrar()
diff --git a/python/mach/mach/terminal.py b/python/mach/mach/terminal.py
new file mode 100644
index 00000000000..cdc3966575e
--- /dev/null
+++ b/python/mach/mach/terminal.py
@@ -0,0 +1,75 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""This file contains code for interacting with terminals.
+
+All the terminal interaction code is consolidated so the complexity can be in
+one place, away from code that is commonly looked at.
+"""
+
+from __future__ import print_function, unicode_literals
+
+import logging
+import sys
+
+
+class LoggingHandler(logging.Handler):
+ """Custom logging handler that works with terminal window dressing.
+
+ This is alternative terminal logging handler which contains smarts for
+ emitting terminal control characters properly. Currently, it has generic
+ support for "footer" elements at the bottom of the screen. Functionality
+ can be added when needed.
+ """
+ def __init__(self):
+ logging.Handler.__init__(self)
+
+ self.fh = sys.stdout
+ self.footer = None
+
+ def flush(self):
+ self.acquire()
+
+ try:
+ self.fh.flush()
+ finally:
+ self.release()
+
+ def emit(self, record):
+ msg = self.format(record)
+
+ if self.footer:
+ self.footer.clear()
+
+ self.fh.write(msg)
+ self.fh.write('\n')
+
+ if self.footer:
+ self.footer.draw()
+
+ # If we don't flush, the footer may not get drawn.
+ self.flush()
+
+
+class TerminalFooter(object):
+ """Represents something drawn on the bottom of a terminal."""
+ def __init__(self, terminal):
+ self.t = terminal
+ self.fh = sys.stdout
+
+ def _clear_lines(self, n):
+ for i in xrange(n):
+ self.fh.write(self.t.move_x(0))
+ self.fh.write(self.t.clear_eol())
+ self.fh.write(self.t.move_up())
+
+ self.fh.write(self.t.move_down())
+ self.fh.write(self.t.move_x(0))
+
+ def clear(self):
+ raise Exception('clear() must be implemented.')
+
+ def draw(self):
+ raise Exception('draw() must be implemented.')
+
diff --git a/python/mach/mach/test/__init__.py b/python/mach/mach/test/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/python/mach/mach/test/__init__.py
diff --git a/python/mach/mach/test/common.py b/python/mach/mach/test/common.py
new file mode 100644
index 00000000000..1c4b1ea90ac
--- /dev/null
+++ b/python/mach/mach/test/common.py
@@ -0,0 +1,40 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+from StringIO import StringIO
+import os
+import unittest
+
+from mach.main import Mach
+from mach.base import CommandContext
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+class TestBase(unittest.TestCase):
+ provider_dir = os.path.join(here, 'providers')
+
+ def _run_mach(self, args, provider_file=None, entry_point=None, context_handler=None):
+ m = Mach(os.getcwd())
+ m.define_category('testing', 'Mach unittest', 'Testing for mach core', 10)
+ m.populate_context_handler = context_handler
+
+ if provider_file:
+ m.load_commands_from_file(os.path.join(self.provider_dir, provider_file))
+
+ if entry_point:
+ m.load_commands_from_entry_point(entry_point)
+
+ stdout = StringIO()
+ stderr = StringIO()
+ stdout.encoding = 'UTF-8'
+ stderr.encoding = 'UTF-8'
+
+ try:
+ result = m.run(args, stdout=stdout, stderr=stderr)
+ except SystemExit:
+ result = None
+
+ return (result, stdout.getvalue(), stderr.getvalue())
diff --git a/python/mach/mach/test/providers/__init__.py b/python/mach/mach/test/providers/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/python/mach/mach/test/providers/__init__.py
diff --git a/python/mach/mach/test/providers/basic.py b/python/mach/mach/test/providers/basic.py
new file mode 100644
index 00000000000..d10856289b1
--- /dev/null
+++ b/python/mach/mach/test/providers/basic.py
@@ -0,0 +1,15 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import unicode_literals
+
+from mach.decorators import (
+ CommandProvider,
+ Command,
+)
+
+@CommandProvider
+class ConditionsProvider(object):
+ @Command('cmd_foo', category='testing')
+ def run_foo(self):
+ pass
diff --git a/python/mach/mach/test/providers/conditions.py b/python/mach/mach/test/providers/conditions.py
new file mode 100644
index 00000000000..a95429752d4
--- /dev/null
+++ b/python/mach/mach/test/providers/conditions.py
@@ -0,0 +1,53 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+from mach.decorators import (
+ CommandProvider,
+ Command,
+)
+
+def is_foo(cls):
+ """Foo must be true"""
+ return cls.foo
+
+def is_bar(cls):
+ """Bar must be true"""
+ return cls.bar
+
+@CommandProvider
+class ConditionsProvider(object):
+ foo = True
+ bar = False
+
+ @Command('cmd_foo', category='testing', conditions=[is_foo])
+ def run_foo(self):
+ pass
+
+ @Command('cmd_bar', category='testing', conditions=[is_bar])
+ def run_bar(self):
+ pass
+
+ @Command('cmd_foobar', category='testing', conditions=[is_foo, is_bar])
+ def run_foobar(self):
+ pass
+
+@CommandProvider
+class ConditionsContextProvider(object):
+ def __init__(self, context):
+ self.foo = context.foo
+ self.bar = context.bar
+
+ @Command('cmd_foo_ctx', category='testing', conditions=[is_foo])
+ def run_foo(self):
+ pass
+
+ @Command('cmd_bar_ctx', category='testing', conditions=[is_bar])
+ def run_bar(self):
+ pass
+
+ @Command('cmd_foobar_ctx', category='testing', conditions=[is_foo, is_bar])
+ def run_foobar(self):
+ pass
diff --git a/python/mach/mach/test/providers/conditions_invalid.py b/python/mach/mach/test/providers/conditions_invalid.py
new file mode 100644
index 00000000000..22284d4dcad
--- /dev/null
+++ b/python/mach/mach/test/providers/conditions_invalid.py
@@ -0,0 +1,16 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+from mach.decorators import (
+ CommandProvider,
+ Command,
+)
+
+@CommandProvider
+class ConditionsProvider(object):
+ @Command('cmd_foo', category='testing', conditions=["invalid"])
+ def run_foo(self):
+ pass
diff --git a/python/mach/mach/test/providers/throw.py b/python/mach/mach/test/providers/throw.py
new file mode 100644
index 00000000000..06bee01eec7
--- /dev/null
+++ b/python/mach/mach/test/providers/throw.py
@@ -0,0 +1,29 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import time
+
+from mach.decorators import (
+ CommandArgument,
+ CommandProvider,
+ Command,
+)
+
+from mach.test.providers import throw2
+
+
+@CommandProvider
+class TestCommandProvider(object):
+ @Command('throw', category='testing')
+ @CommandArgument('--message', '-m', default='General Error')
+ def throw(self, message):
+ raise Exception(message)
+
+ @Command('throw_deep', category='testing')
+ @CommandArgument('--message', '-m', default='General Error')
+ def throw_deep(self, message):
+ throw2.throw_deep(message)
+
diff --git a/python/mach/mach/test/providers/throw2.py b/python/mach/mach/test/providers/throw2.py
new file mode 100644
index 00000000000..af0a23fcfe7
--- /dev/null
+++ b/python/mach/mach/test/providers/throw2.py
@@ -0,0 +1,13 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This file exists to trigger the differences in mach error reporting between
+# exceptions that occur in mach command modules themselves and in the things
+# they call.
+
+def throw_deep(message):
+ return throw_real(message)
+
+def throw_real(message):
+ raise Exception(message)
diff --git a/python/mach/mach/test/test_conditions.py b/python/mach/mach/test/test_conditions.py
new file mode 100644
index 00000000000..532a8316062
--- /dev/null
+++ b/python/mach/mach/test/test_conditions.py
@@ -0,0 +1,82 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import os
+
+from mach.base import MachError
+from mach.main import Mach
+from mach.test.common import TestBase
+
+from mozunit import main
+
+
+def _populate_context(context, key=None):
+ if key is None:
+ return
+ if key == 'foo':
+ return True
+ if key == 'bar':
+ return False
+ raise AttributeError(key)
+
+class TestConditions(TestBase):
+ """Tests for conditionally filtering commands."""
+
+ def _run_mach(self, args, context_handler=None):
+ return TestBase._run_mach(self, args, 'conditions.py',
+ context_handler=context_handler)
+
+
+ def test_conditions_pass(self):
+ """Test that a command which passes its conditions is runnable."""
+
+ self.assertEquals((0, '', ''), self._run_mach(['cmd_foo']))
+ self.assertEquals((0, '', ''), self._run_mach(['cmd_foo_ctx'], _populate_context))
+
+ def test_invalid_context_message(self):
+ """Test that commands which do not pass all their conditions
+ print the proper failure message."""
+
+ def is_bar():
+ """Bar must be true"""
+ fail_conditions = [is_bar]
+
+ for name in ('cmd_bar', 'cmd_foobar'):
+ result, stdout, stderr = self._run_mach([name])
+ self.assertEquals(1, result)
+
+ fail_msg = Mach._condition_failed_message(name, fail_conditions)
+ self.assertEquals(fail_msg.rstrip(), stdout.rstrip())
+
+ for name in ('cmd_bar_ctx', 'cmd_foobar_ctx'):
+ result, stdout, stderr = self._run_mach([name], _populate_context)
+ self.assertEquals(1, result)
+
+ fail_msg = Mach._condition_failed_message(name, fail_conditions)
+ self.assertEquals(fail_msg.rstrip(), stdout.rstrip())
+
+ def test_invalid_type(self):
+ """Test that a condition which is not callable raises an exception."""
+
+ m = Mach(os.getcwd())
+ m.define_category('testing', 'Mach unittest', 'Testing for mach core', 10)
+ self.assertRaises(MachError, m.load_commands_from_file,
+ os.path.join(self.provider_dir, 'conditions_invalid.py'))
+
+ def test_help_message(self):
+ """Test that commands that are not runnable do not show up in help."""
+
+ result, stdout, stderr = self._run_mach(['help'], _populate_context)
+ self.assertIn('cmd_foo', stdout)
+ self.assertNotIn('cmd_bar', stdout)
+ self.assertNotIn('cmd_foobar', stdout)
+ self.assertIn('cmd_foo_ctx', stdout)
+ self.assertNotIn('cmd_bar_ctx', stdout)
+ self.assertNotIn('cmd_foobar_ctx', stdout)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/python/mach/mach/test/test_config.py b/python/mach/mach/test/test_config.py
new file mode 100644
index 00000000000..cebd47a7d6c
--- /dev/null
+++ b/python/mach/mach/test/test_config.py
@@ -0,0 +1,264 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import unicode_literals
+
+import sys
+import unittest
+
+from mozfile.mozfile import NamedTemporaryFile
+
+from mach.config import (
+ AbsolutePathType,
+ BooleanType,
+ ConfigProvider,
+ ConfigSettings,
+ IntegerType,
+ PathType,
+ PositiveIntegerType,
+ RelativePathType,
+ StringType,
+)
+
+from mozunit import main
+
+
+if sys.version_info[0] == 3:
+ str_type = str
+else:
+ str_type = basestring
+
+CONFIG1 = r"""
+[foo]
+
+bar = bar_value
+baz = /baz/foo.c
+"""
+
+CONFIG2 = r"""
+[foo]
+
+bar = value2
+"""
+
+class Provider1(ConfigProvider):
+ @classmethod
+ def _register_settings(cls):
+ cls.register_setting('foo', 'bar', StringType)
+ cls.register_setting('foo', 'baz', AbsolutePathType)
+
+Provider1.register_settings()
+
+class ProviderDuplicate(ConfigProvider):
+ @classmethod
+ def _register_settings(cls):
+ cls.register_setting('dupesect', 'foo', StringType)
+ cls.register_setting('dupesect', 'foo', StringType)
+
+class TestConfigProvider(unittest.TestCase):
+ def test_construct(self):
+ s = Provider1.config_settings
+
+ self.assertEqual(len(s), 1)
+ self.assertIn('foo', s)
+
+ self.assertEqual(len(s['foo']), 2)
+ self.assertIn('bar', s['foo'])
+ self.assertIn('baz', s['foo'])
+
+ def test_duplicate_option(self):
+ with self.assertRaises(Exception):
+ ProviderDuplicate.register_settings()
+
+
+class Provider2(ConfigProvider):
+ @classmethod
+ def _register_settings(cls):
+ cls.register_setting('a', 'string', StringType)
+ cls.register_setting('a', 'boolean', BooleanType)
+ cls.register_setting('a', 'pos_int', PositiveIntegerType)
+ cls.register_setting('a', 'int', IntegerType)
+ cls.register_setting('a', 'abs_path', AbsolutePathType)
+ cls.register_setting('a', 'rel_path', RelativePathType)
+ cls.register_setting('a', 'path', PathType)
+
+Provider2.register_settings()
+
+class TestConfigSettings(unittest.TestCase):
+ def test_empty(self):
+ s = ConfigSettings()
+
+ self.assertEqual(len(s), 0)
+ self.assertNotIn('foo', s)
+
+ def test_simple(self):
+ s = ConfigSettings()
+ s.register_provider(Provider1)
+
+ self.assertEqual(len(s), 1)
+ self.assertIn('foo', s)
+
+ foo = s['foo']
+ foo = s.foo
+
+ self.assertEqual(len(foo), 2)
+
+ self.assertIn('bar', foo)
+ self.assertIn('baz', foo)
+
+ foo['bar'] = 'value1'
+ self.assertEqual(foo['bar'], 'value1')
+ self.assertEqual(foo['bar'], 'value1')
+
+ def test_assignment_validation(self):
+ s = ConfigSettings()
+ s.register_provider(Provider2)
+
+ a = s.a
+
+ # Assigning an undeclared setting raises.
+ with self.assertRaises(KeyError):
+ a.undefined = True
+
+ with self.assertRaises(KeyError):
+ a['undefined'] = True
+
+ # Basic type validation.
+ a.string = 'foo'
+ a.string = 'foo'
+
+ with self.assertRaises(TypeError):
+ a.string = False
+
+ a.boolean = True
+ a.boolean = False
+
+ with self.assertRaises(TypeError):
+ a.boolean = 'foo'
+
+ a.pos_int = 5
+ a.pos_int = 0
+
+ with self.assertRaises(ValueError):
+ a.pos_int = -1
+
+ with self.assertRaises(TypeError):
+ a.pos_int = 'foo'
+
+ a.int = 5
+ a.int = 0
+ a.int = -5
+
+ with self.assertRaises(TypeError):
+ a.int = 1.24
+
+ with self.assertRaises(TypeError):
+ a.int = 'foo'
+
+ a.abs_path = '/home/gps'
+
+ with self.assertRaises(ValueError):
+ a.abs_path = 'home/gps'
+
+ a.rel_path = 'home/gps'
+ a.rel_path = './foo/bar'
+ a.rel_path = 'foo.c'
+
+ with self.assertRaises(ValueError):
+ a.rel_path = '/foo/bar'
+
+ a.path = '/home/gps'
+ a.path = 'foo.c'
+ a.path = 'foo/bar'
+ a.path = './foo'
+
+ def test_retrieval_type(self):
+ s = ConfigSettings()
+ s.register_provider(Provider2)
+
+ a = s.a
+
+ a.string = 'foo'
+ a.boolean = True
+ a.pos_int = 12
+ a.int = -4
+ a.abs_path = '/home/gps'
+ a.rel_path = 'foo.c'
+ a.path = './foo/bar'
+
+ self.assertIsInstance(a.string, str_type)
+ self.assertIsInstance(a.boolean, bool)
+ self.assertIsInstance(a.pos_int, int)
+ self.assertIsInstance(a.int, int)
+ self.assertIsInstance(a.abs_path, str_type)
+ self.assertIsInstance(a.rel_path, str_type)
+ self.assertIsInstance(a.path, str_type)
+
+ def test_file_reading_single(self):
+ temp = NamedTemporaryFile(mode='wt')
+ temp.write(CONFIG1)
+ temp.flush()
+
+ s = ConfigSettings()
+ s.register_provider(Provider1)
+
+ s.load_file(temp.name)
+
+ self.assertEqual(s.foo.bar, 'bar_value')
+
+ def test_file_reading_multiple(self):
+ """Loading multiple files has proper overwrite behavior."""
+ temp1 = NamedTemporaryFile(mode='wt')
+ temp1.write(CONFIG1)
+ temp1.flush()
+
+ temp2 = NamedTemporaryFile(mode='wt')
+ temp2.write(CONFIG2)
+ temp2.flush()
+
+ s = ConfigSettings()
+ s.register_provider(Provider1)
+
+ s.load_files([temp1.name, temp2.name])
+
+ self.assertEqual(s.foo.bar, 'value2')
+
+ def test_file_reading_missing(self):
+ """Missing files should silently be ignored."""
+
+ s = ConfigSettings()
+
+ s.load_file('/tmp/foo.ini')
+
+ def test_file_writing(self):
+ s = ConfigSettings()
+ s.register_provider(Provider2)
+
+ s.a.string = 'foo'
+ s.a.boolean = False
+
+ temp = NamedTemporaryFile('wt')
+ s.write(temp)
+ temp.flush()
+
+ s2 = ConfigSettings()
+ s2.register_provider(Provider2)
+
+ s2.load_file(temp.name)
+
+ self.assertEqual(s.a.string, s2.a.string)
+ self.assertEqual(s.a.boolean, s2.a.boolean)
+
+ def test_write_pot(self):
+ s = ConfigSettings()
+ s.register_provider(Provider1)
+ s.register_provider(Provider2)
+
+ # Just a basic sanity test.
+ temp = NamedTemporaryFile('wt')
+ s.write_pot(temp)
+ temp.flush()
+
+
+if __name__ == '__main__':
+ main()
diff --git a/python/mach/mach/test/test_entry_point.py b/python/mach/mach/test/test_entry_point.py
new file mode 100644
index 00000000000..5bd2c279d45
--- /dev/null
+++ b/python/mach/mach/test/test_entry_point.py
@@ -0,0 +1,60 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import unicode_literals
+
+import imp
+import os
+import sys
+
+from mach.base import MachError
+from mach.test.common import TestBase
+from mock import patch
+
+from mozunit import main
+
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+class Entry():
+ """Stub replacement for pkg_resources.EntryPoint"""
+ def __init__(self, providers):
+ self.providers = providers
+
+ def load(self):
+ def _providers():
+ return self.providers
+ return _providers
+
+class TestEntryPoints(TestBase):
+ """Test integrating with setuptools entry points"""
+ provider_dir = os.path.join(here, 'providers')
+
+ def _run_mach(self):
+ return TestBase._run_mach(self, ['help'], entry_point='mach.providers')
+
+ @patch('pkg_resources.iter_entry_points')
+ def test_load_entry_point_from_directory(self, mock):
+ # Ensure parent module is present otherwise we'll (likely) get
+ # an error due to unknown parent.
+ if b'mach.commands' not in sys.modules:
+ mod = imp.new_module(b'mach.commands')
+ sys.modules[b'mach.commands'] = mod
+
+ mock.return_value = [Entry(['providers'])]
+ # Mach error raised due to conditions_invalid.py
+ with self.assertRaises(MachError):
+ self._run_mach()
+
+ @patch('pkg_resources.iter_entry_points')
+ def test_load_entry_point_from_file(self, mock):
+ mock.return_value = [Entry([os.path.join('providers', 'basic.py')])]
+
+ result, stdout, stderr = self._run_mach()
+ self.assertIsNone(result)
+ self.assertIn('cmd_foo', stdout)
+
+
+# Not enabled in automation because tests are failing.
+#if __name__ == '__main__':
+# main()
diff --git a/python/mach/mach/test/test_error_output.py b/python/mach/mach/test/test_error_output.py
new file mode 100644
index 00000000000..25553f96bc4
--- /dev/null
+++ b/python/mach/mach/test/test_error_output.py
@@ -0,0 +1,39 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+from mach.main import (
+ COMMAND_ERROR,
+ MODULE_ERROR
+)
+from mach.test.common import TestBase
+
+from mozunit import main
+
+
+class TestErrorOutput(TestBase):
+
+ def _run_mach(self, args):
+ return TestBase._run_mach(self, args, 'throw.py')
+
+ def test_command_error(self):
+ result, stdout, stderr = self._run_mach(['throw', '--message',
+ 'Command Error'])
+
+ self.assertEqual(result, 1)
+
+ self.assertIn(COMMAND_ERROR, stdout)
+
+ def test_invoked_error(self):
+ result, stdout, stderr = self._run_mach(['throw_deep', '--message',
+ 'Deep stack'])
+
+ self.assertEqual(result, 1)
+
+ self.assertIn(MODULE_ERROR, stdout)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/python/mach/mach/test/test_logger.py b/python/mach/mach/test/test_logger.py
new file mode 100644
index 00000000000..05592845e7f
--- /dev/null
+++ b/python/mach/mach/test/test_logger.py
@@ -0,0 +1,47 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, unicode_literals
+
+import logging
+import time
+import unittest
+
+from mach.logging import StructuredHumanFormatter
+
+from mozunit import main
+
+
+class DummyLogger(logging.Logger):
+ def __init__(self, cb):
+ logging.Logger.__init__(self, 'test')
+
+ self._cb = cb
+
+ def handle(self, record):
+ self._cb(record)
+
+
+class TestStructuredHumanFormatter(unittest.TestCase):
+ def test_non_ascii_logging(self):
+ # Ensures the formatter doesn't choke when non-ASCII characters are
+ # present in printed parameters.
+ formatter = StructuredHumanFormatter(time.time())
+
+ def on_record(record):
+ result = formatter.format(record)
+ relevant = result[9:]
+
+ self.assertEqual(relevant, 'Test: s\xe9curit\xe9')
+
+ logger = DummyLogger(on_record)
+
+ value = 's\xe9curit\xe9'
+
+ logger.log(logging.INFO, 'Test: {utf}',
+ extra={'action': 'action', 'params': {'utf': value}})
+
+
+if __name__ == '__main__':
+ main()
diff --git a/python/mach/setup.py b/python/mach/setup.py
new file mode 100644
index 00000000000..511a6a32277
--- /dev/null
+++ b/python/mach/setup.py
@@ -0,0 +1,38 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+try:
+ from setuptools import setup
+except:
+ from distutils.core import setup
+
+
+VERSION = '0.3'
+
+README = open('README.rst').read()
+
+setup(
+ name='mach',
+ description='Generic command line command dispatching framework.',
+ long_description=README,
+ license='MPL 2.0',
+ author='Gregory Szorc',
+ author_email='gregory.szorc@gmail.com',
+ url='https://developer.mozilla.org/en-US/docs/Developer_Guide/mach',
+ packages=['mach'],
+ version=VERSION,
+ classifiers=[
+ 'Environment :: Console',
+ 'Development Status :: 3 - Alpha',
+ 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)',
+ 'Natural Language :: English',
+ ],
+ install_requires=[
+ 'blessings',
+ 'mozfile',
+ 'mozprocess',
+ ],
+ tests_require=['mock'],
+)
+
diff --git a/python/mach_bootstrap.py b/python/mach_bootstrap.py
new file mode 100644
index 00000000000..3418b87d057
--- /dev/null
+++ b/python/mach_bootstrap.py
@@ -0,0 +1,100 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import print_function, unicode_literals
+
+import os
+import platform
+import sys
+
+SEARCH_PATHS = [
+ "python/mach",
+ "python/toml",
+]
+
+# Individual files providing mach commands.
+MACH_MODULES = [
+ 'python/servo/bootstrap_commands.py',
+ 'python/servo/build_commands.py',
+ 'python/servo/testing_commands.py',
+ 'python/servo/post_build_commands.py',
+ 'python/servo/devenv_commands.py',
+]
+
+
+CATEGORIES = {
+ 'bootstrap': {
+ 'short': 'Bootstrap Commands',
+ 'long': 'Bootstrap the build system',
+ 'priority': 90,
+ },
+ 'build': {
+ 'short': 'Build Commands',
+ 'long': 'Interact with the build system',
+ 'priority': 80,
+ },
+ 'post-build': {
+ 'short': 'Post-build Commands',
+ 'long': 'Common actions performed after completing a build.',
+ 'priority': 70,
+ },
+ 'testing': {
+ 'short': 'Testing',
+ 'long': 'Run tests.',
+ 'priority': 60,
+ },
+ 'devenv': {
+ 'short': 'Development Environment',
+ 'long': 'Set up and configure your development environment.',
+ 'priority': 50,
+ },
+ 'build-dev': {
+ 'short': 'Low-level Build System Interaction',
+ 'long': 'Interact with specific parts of the build system.',
+ 'priority': 20,
+ },
+ 'misc': {
+ 'short': 'Potpourri',
+ 'long': 'Potent potables and assorted snacks.',
+ 'priority': 10,
+ },
+ 'disabled': {
+ 'short': 'Disabled',
+ 'long': 'The disabled commands are hidden by default. Use -v to display them. These commands are unavailable for your current context, run "mach <command>" to see why.',
+ 'priority': 0,
+ }
+}
+
+
+def bootstrap(topdir):
+ topdir = os.path.abspath(topdir)
+
+ # Ensure we are running Python 2.7+. We put this check here so we generate a
+ # user-friendly error message rather than a cryptic stack trace on module
+ # import.
+ if sys.version_info[0] != 2 or sys.version_info[1] < 7:
+ print('Python 2.7 or above (but not Python 3) is required to run mach.')
+ print('You are running Python', platform.python_version())
+ sys.exit(1)
+
+ def populate_context(context, key=None):
+ if key is None:
+ return
+ if key == 'topdir':
+ return topdir
+ raise AttributeError(key)
+
+ sys.path[0:0] = [os.path.join(topdir, path) for path in SEARCH_PATHS]
+ import mach.main
+ mach = mach.main.Mach(os.getcwd())
+ mach.populate_context_handler = populate_context
+
+ for category, meta in CATEGORIES.items():
+ mach.define_category(category, meta['short'], meta['long'],
+ meta['priority'])
+
+ for path in MACH_MODULES:
+ mach.load_commands_from_file(os.path.join(topdir, path))
+
+ return mach
diff --git a/python/servo/__init__.py b/python/servo/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/python/servo/__init__.py
diff --git a/python/servo/bootstrap_commands.py b/python/servo/bootstrap_commands.py
new file mode 100644
index 00000000000..13a411f7008
--- /dev/null
+++ b/python/servo/bootstrap_commands.py
@@ -0,0 +1,153 @@
+from __future__ import print_function, unicode_literals
+
+import os
+import os.path as path
+import shutil
+import subprocess
+import sys
+import tarfile
+import urllib
+
+from mach.decorators import (
+ CommandArgument,
+ CommandProvider,
+ Command,
+)
+
+from servo.command_base import CommandBase, cd
+
+def host_triple():
+ os_type = subprocess.check_output(["uname", "-s"]).strip().lower()
+ if os_type == "linux":
+ os_type = "unknown-linux-gnu"
+ elif os_type == "darwin":
+ os_type = "apple-darwin"
+ elif os_type == "android":
+ os_type == "linux-androideabi"
+ else:
+ os_type == "unknown"
+
+ cpu_type = subprocess.check_output(["uname", "-m"]).strip().lower()
+ if cpu_type in ["i386", "i486", "i686", "i768", "x86"]:
+ cpu_type = "i686"
+ elif cpu_type in ["x86_64", "x86-64", "x64", "amd64"]:
+ cpu_type = "x86_64"
+ elif cpu_type == "arm":
+ cpu_type = "arm"
+ else:
+ cpu_type = "unknown"
+
+ return "%s-%s" % (cpu_type, os_type)
+
+def download(desc, src, dst):
+ recved = [0]
+ def report(count, bsize, fsize):
+ recved[0] += bsize
+ pct = recved[0] * 100.0 / fsize
+ print("\rDownloading %s: %5.1f%%" % (desc, pct), end="")
+ sys.stdout.flush()
+
+ urllib.urlretrieve(src, dst, report)
+ print()
+
+def extract(src, dst, movedir=None):
+ tarfile.open(src).extractall(dst)
+
+ if movedir:
+ for f in os.listdir(movedir):
+ frm = path.join(movedir, f)
+ to = path.join(dst, f)
+ os.rename(frm, to)
+ os.rmdir(movedir)
+
+ os.remove(src)
+
+@CommandProvider
+class MachCommands(CommandBase):
+ @Command('env',
+ description='Print environment setup commands',
+ category='bootstrap')
+ def env(self):
+ env = self.build_env()
+ print("export PATH=%s" % env["PATH"])
+ if sys.platform == "darwin":
+ print("export DYLD_LIBRARY_PATH=%s" % env["DYLD_LIBRARY_PATH"])
+ else:
+ print("export LD_LIBRARY_PATH=%s" % env["LD_LIBRARY_PATH"])
+
+ @Command('bootstrap-rust',
+ description='Download the Rust compiler snapshot',
+ category='bootstrap')
+ @CommandArgument('--force', '-f',
+ action='store_true',
+ help='Force download even if a snapshot already exists')
+ def bootstrap_rustc(self, force=False):
+ rust_dir = path.join(self.context.topdir, "rust")
+ if not force and path.exists(path.join(rust_dir, "bin", "rustc")):
+ print("Snapshot Rust compiler already downloaded.", end=" ")
+ print("Use |bootstrap_rust --force| to download again.")
+ return
+
+ if path.isdir(rust_dir):
+ shutil.rmtree(rust_dir)
+ os.mkdir(rust_dir)
+
+ snapshot_hash = open(path.join(self.context.topdir, "rust-snapshot-hash")).read().strip()
+ snapshot_path = "%s-%s.tar.gz" % (snapshot_hash, host_triple())
+ snapshot_url = "https://servo-rust.s3.amazonaws.com/%s" % snapshot_path
+ tgz_file = path.join(rust_dir, path.basename(snapshot_path))
+
+ download("Rust snapshot", snapshot_url, tgz_file)
+
+ print("Extracting Rust snapshot...")
+ snap_dir = path.join(rust_dir,
+ path.basename(tgz_file).replace(".tar.gz", ""))
+ extract(tgz_file, rust_dir, movedir=snap_dir)
+ print("Snapshot Rust ready.")
+
+ @Command('bootstrap-cargo',
+ description='Download the Cargo build tool',
+ category='bootstrap')
+ @CommandArgument('--force', '-f',
+ action='store_true',
+ help='Force download even if cargo already exists')
+ def bootstrap_cargo(self, force=False):
+ cargo_dir = path.join(self.context.topdir, "cargo")
+ if not force and path.exists(path.join(cargo_dir, "bin", "cargo")):
+ print("Cargo already downloaded.", end=" ")
+ print("Use |bootstrap_cargo --force| to download again.")
+ return
+
+ if path.isdir(cargo_dir):
+ shutil.rmtree(cargo_dir)
+ os.mkdir(cargo_dir)
+
+ tgz_file = "cargo-nightly-%s.tar.gz" % host_triple()
+ nightly_url = "http://static.rust-lang.org/cargo-dist/%s" % tgz_file
+
+ download("Cargo nightly", nightly_url, tgz_file)
+
+ print("Extracting Cargo nightly...")
+ nightly_dir = path.join(cargo_dir,
+ path.basename(tgz_file).replace(".tar.gz", ""))
+ extract(tgz_file, cargo_dir, movedir=nightly_dir)
+ print("Cargo ready.")
+
+ @Command('update-submodules',
+ description='Update submodules',
+ category='bootstrap')
+ def update_submodules(self):
+ submodules = subprocess.check_output(["git", "submodule", "status"])
+ for line in submodules.split('\n'):
+ components = line.strip().split(' ')
+ if len(components) > 1:
+ module_path = components[1]
+ if path.exists(module_path):
+ with cd(module_path):
+ output = subprocess.check_output(["git", "status", "--porcelain"])
+ if len(output) != 0:
+ print("error: submodule %s is not clean" % module_path)
+ print("\nClean the submodule and try again.")
+ return 1
+ subprocess.check_call(["git", "submodule", "--quiet", "sync", "--recursive"])
+ subprocess.check_call(["git", "submodule", "update", "--init", "--recursive"])
diff --git a/python/servo/build_commands.py b/python/servo/build_commands.py
new file mode 100644
index 00000000000..b0f4ab95b64
--- /dev/null
+++ b/python/servo/build_commands.py
@@ -0,0 +1,85 @@
+from __future__ import print_function, unicode_literals
+
+import json
+import os
+import os.path as path
+import shutil
+import subprocess
+import sys
+import tarfile
+from time import time
+import urllib
+
+from mach.registrar import Registrar
+from mach.decorators import (
+ CommandArgument,
+ CommandProvider,
+ Command,
+)
+
+from servo.command_base import CommandBase, cd
+
+@CommandProvider
+class MachCommands(CommandBase):
+ @Command('build',
+ description='Build Servo',
+ category='build')
+ @CommandArgument('--target', '-t',
+ default=None,
+ help='Cross compile for given target platform')
+ @CommandArgument('--release', '-r',
+ action='store_true',
+ help='Build in release mode')
+ @CommandArgument('--jobs', '-j',
+ default=None,
+ help='Number of jobs to run in parallel')
+ def build(self, target, release=False, jobs=None):
+ self.ensure_bootstrapped()
+
+ opts = []
+ if release:
+ opts += ["--release"]
+ if jobs is not None:
+ opts += ["-j", jobs]
+
+ build_start = time()
+ subprocess.check_call(["cargo", "build"] + opts, env=self.build_env())
+ elapsed = time() - build_start
+
+ print("Build completed in %0.2fs" % elapsed)
+
+ @Command('build-cef',
+ description='Build the Chromium Embedding Framework library',
+ category='build')
+ @CommandArgument('--jobs', '-j',
+ default=None,
+ help='Number of jobs to run in parallel')
+ def build_cef(self, jobs=None):
+ self.ensure_bootstrapped()
+
+ ret = None
+ opts = []
+ if jobs is not None:
+ opts += ["-j", jobs]
+
+ build_start = time()
+ with cd(path.join("ports", "cef")):
+ ret = subprocess.call(["cargo", "build"], env=self.build_env())
+ elapsed = time() - build_start
+
+ print("CEF build completed in %0.2fs" % elapsed)
+
+ return ret
+
+ @Command('build-tests',
+ description='Build the Servo test suites',
+ category='build')
+ @CommandArgument('--jobs', '-j',
+ default=None,
+ help='Number of jobs to run in parallel')
+ def build_tests(self, jobs=None):
+ self.ensure_bootstrapped()
+ opts = []
+ if jobs is not None:
+ opts += ["-j", jobs]
+ subprocess.check_call(["cargo", "test", "--no-run"], env=self.build_env())
diff --git a/python/servo/command_base.py b/python/servo/command_base.py
new file mode 100644
index 00000000000..5d15494a4e7
--- /dev/null
+++ b/python/servo/command_base.py
@@ -0,0 +1,94 @@
+import os
+from os import path
+import subprocess
+import sys
+import toml
+
+from mach.registrar import Registrar
+
+class cd:
+ """Context manager for changing the current working directory"""
+ def __init__(self, newPath):
+ self.newPath = newPath
+
+ def __enter__(self):
+ self.savedPath = os.getcwd()
+ os.chdir(self.newPath)
+
+ def __exit__(self, etype, value, traceback):
+ os.chdir(self.savedPath)
+
+class CommandBase(object):
+ """Base class for mach command providers.
+
+ This mostly handles configuration management, such as .servobuild."""
+
+ def __init__(self, context):
+ self.context = context
+
+ if not hasattr(self.context, "bootstrapped"):
+ self.context.bootstrapped = False
+
+ config_path = path.join(context.topdir, ".servobuild")
+ if path.exists(config_path):
+ self.config = toml.loads(open(config_path).read())
+ else:
+ self.config = {}
+
+ # Handle missing/default items
+ self.config.setdefault("tools", {})
+ self.config["tools"].setdefault("system-rust", False)
+ self.config["tools"].setdefault("system-cargo", False)
+ self.config["tools"].setdefault("rust-root", "")
+ self.config["tools"].setdefault("cargo-root", "")
+ if not self.config["tools"]["system-rust"]:
+ self.config["tools"]["rust-root"] = path.join(context.topdir, "rust")
+ if not self.config["tools"]["system-cargo"]:
+ self.config["tools"]["cargo-root"] = path.join(context.topdir, "cargo")
+
+ def build_env(self):
+ """Return an extended environment dictionary."""
+ env = os.environ.copy()
+ extra_path = []
+ extra_lib = []
+ if not self.config["tools"]["system-rust"] or self.config["tools"]["rust-root"]:
+ extra_path += [path.join(self.config["tools"]["rust-root"], "bin")]
+ extra_lib += [path.join(self.config["tools"]["rust-root"], "lib")]
+ if not self.config["tools"]["system-cargo"] or self.config["tools"]["cargo-root"]:
+ extra_path += [path.join(self.config["tools"]["cargo-root"], "bin")]
+
+ if extra_path:
+ env["PATH"] = "%s%s%s" % (os.pathsep.join(extra_path), os.pathsep, env["PATH"])
+ if extra_lib:
+ if sys.platform == "darwin":
+ env["DYLD_LIBRARY_PATH"] = "%s%s%s" % \
+ (os.pathsep.join(extra_lib),
+ os.pathsep,
+ env.get("DYLD_LIBRARY_PATH", ""))
+ else:
+ env["LD_LIBRARY_PATH"] = "%s%s%s" % \
+ (os.pathsep.join(extra_lib),
+ os.pathsep,
+ env.get("LD_LIBRARY_PATH", ""))
+
+ return env
+
+ def ensure_bootstrapped(self):
+ if self.context.bootstrapped: return
+
+ submodules = subprocess.check_output(["git", "submodule", "status"])
+ for line in submodules.split('\n'):
+ components = line.strip().split(' ')
+ if len(components) > 1 and components[0].startswith('-'):
+ module_path = components[1]
+ subprocess.check_call(["git", "submodule", "update",
+ "--init", "--recursive", "--", module_path])
+
+ if not self.config["tools"]["system-rust"] and \
+ not path.exists(path.join(self.context.topdir, "rust", "bin", "rustc")):
+ Registrar.dispatch("bootstrap-rust", context=self.context)
+ if not self.config["tools"]["system-cargo"] and \
+ not path.exists(path.join(self.context.topdir, "cargo", "bin", "cargo")):
+ Registrar.dispatch("bootstrap-cargo", context=self.context)
+
+ self.context.bootstrapped = True
diff --git a/python/servo/devenv_commands.py b/python/servo/devenv_commands.py
new file mode 100644
index 00000000000..1c5c8ee24b5
--- /dev/null
+++ b/python/servo/devenv_commands.py
@@ -0,0 +1,32 @@
+from __future__ import print_function, unicode_literals
+
+import json
+import os
+import os.path as path
+import shutil
+import subprocess
+import sys
+import tarfile
+from time import time
+import urllib
+
+from mach.registrar import Registrar
+from mach.decorators import (
+ CommandArgument,
+ CommandProvider,
+ Command,
+)
+
+from servo.command_base import CommandBase
+
+@CommandProvider
+class MachCommands(CommandBase):
+ @Command('cargo',
+ description='Run Cargo',
+ category='devenv',
+ allow_all_args=True)
+ @CommandArgument('params', default=None, nargs='...',
+ help="Command-line arguments to be passed through to Cervo")
+ def run(self, params):
+ return subprocess.call(["cargo"] + params,
+ env=self.build_env())
diff --git a/python/servo/post_build_commands.py b/python/servo/post_build_commands.py
new file mode 100644
index 00000000000..276a772c787
--- /dev/null
+++ b/python/servo/post_build_commands.py
@@ -0,0 +1,44 @@
+from __future__ import print_function, unicode_literals
+
+import json
+import os
+import os.path as path
+import shutil
+import subprocess
+import sys
+import tarfile
+from time import time
+import urllib
+
+from mach.registrar import Registrar
+from mach.decorators import (
+ CommandArgument,
+ CommandProvider,
+ Command,
+)
+
+from servo.command_base import CommandBase
+
+@CommandProvider
+class MachCommands(CommandBase):
+ @Command('run',
+ description='Run Servo',
+ category='post-build',
+ allow_all_args=True)
+ @CommandArgument('params', default=None, nargs='...',
+ help="Command-line arguments to be passed through to Servo")
+ def run(self, params):
+ subprocess.check_call([path.join("target", "servo")] + params,
+ env=self.build_env())
+
+ @Command('doc',
+ description='Generate documentation',
+ category='post-build',
+ allow_all_args=True)
+ @CommandArgument('params', default=None, nargs='...',
+ help="Command-line arguments to be passed through to cargo doc")
+ def doc(self, params):
+ self.ensure_bootstrapped()
+ return subprocess.call(["cargo", "doc"] + params,
+ env=self.build_env())
+
diff --git a/python/servo/testing_commands.py b/python/servo/testing_commands.py
new file mode 100644
index 00000000000..19ecc4ef4cf
--- /dev/null
+++ b/python/servo/testing_commands.py
@@ -0,0 +1,122 @@
+from __future__ import print_function, unicode_literals
+
+import json
+import os
+import os.path as path
+import shutil
+import subprocess
+import sys
+import tarfile
+from time import time
+import urllib
+
+from mach.registrar import Registrar
+from mach.decorators import (
+ CommandArgument,
+ CommandProvider,
+ Command,
+)
+
+from servo.command_base import CommandBase
+import tidy
+
+@CommandProvider
+class MachCommands(CommandBase):
+ def __init__(self, context):
+ CommandBase.__init__(self, context)
+ if not hasattr(self.context, "built_tests"):
+ self.context.built_tests = False
+
+ def ensure_built_tests(self):
+ if self.context.built_tests: return
+ Registrar.dispatch('build-tests', context=self.context)
+ self.context.built_tests = True
+
+ def find_test(self, prefix):
+ candidates = [f for f in os.listdir(path.join(self.context.topdir, "target"))
+ if f.startswith(prefix + "-")]
+ if candidates:
+ return path.join(self.context.topdir, "target", candidates[0])
+ return None
+
+ def run_test(self, prefix, args=[]):
+ t = self.find_test(prefix)
+ if t:
+ return subprocess.call([t] + args, env=self.build_env())
+
+ @Command('test',
+ description='Run all Servo tests',
+ category='testing')
+ def test(self):
+ test_start = time()
+ for t in ["tidy", "unit", "ref", "content", "wpt"]:
+ Registrar.dispatch("test-%s" % t, context=self.context)
+ elapsed = time() - test_start
+
+ print("Tests completed in %0.2fs" % elapsed)
+
+ @Command('test-unit',
+ description='Run libservo unit tests',
+ category='testing')
+ def test_unit(self):
+ self.ensure_bootstrapped()
+ self.ensure_built_tests()
+ return self.run_test("servo")
+
+ @Command('test-ref',
+ description='Run the reference tests',
+ category='testing')
+ @CommandArgument('--kind', '-k', default=None)
+ def test_ref(self, kind=None):
+ self.ensure_bootstrapped()
+ self.ensure_built_tests()
+
+ kinds = ["cpu", "gpu"] if kind is None else [kind]
+ test_path = path.join(self.context.topdir, "tests", "ref")
+ error = False
+
+ test_start = time()
+ for k in kinds:
+ print("Running %s reftests..." % k)
+ ret = self.run_test("reftest", [k, test_path])
+ error = error or ret != 0
+ elapsed = time() - test_start
+
+ print("Reference tests completed in %0.2fs" % elapsed)
+
+ if error: return 1
+
+ @Command('test-content',
+ description='Run the content tests',
+ category='testing')
+ def test_content(self):
+ self.ensure_bootstrapped()
+ self.ensure_built_tests()
+
+ test_path = path.join(self.context.topdir, "tests", "content")
+ test_start = time()
+ ret = self.run_test("contenttest", ["--source-dir=%s" % test_path])
+ elapsed = time() - test_start
+
+ print("Content tests completed in %0.2fs" % elapsed)
+ return ret
+
+ @Command('test-tidy',
+ description='Run the source code tidiness check',
+ category='testing')
+ def test_tidy(self):
+ errors = 0
+ for p in ["src", "components"]:
+ ret = tidy.scan(path.join(self.context.topdir, p))
+ if ret != 0: errors = 1
+ return errors
+
+ @Command('test-wpt',
+ description='Run the web platform tests',
+ category='testing',
+ allow_all_args=True)
+ @CommandArgument('params', default=None, nargs='...',
+ help="Command-line arguments to be passed through to wpt/run.sh")
+ def test_wpt(self, params):
+ return subprocess.call(["bash", path.join("tests", "wpt", "run.sh")] + params,
+ env=self.build_env())
diff --git a/python/tidy.py b/python/tidy.py
new file mode 100644
index 00000000000..04743433dbf
--- /dev/null
+++ b/python/tidy.py
@@ -0,0 +1,91 @@
+# Copyright 2013 The Servo Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+#!/usr/bin/env python
+
+import os
+from licenseck import check_license
+
+# FIXME(#3242): Don't use globals
+err = 0
+
+
+def report_error_name_no(name, no, s):
+ global err
+ print("%s:%d: %s" % (name, no, s))
+ err = 1
+
+
+def do_license_check(name, contents):
+ if not check_license(name, contents):
+ report_error_name_no(name, 1, "incorrect license")
+
+
+def do_whitespace_check(name, contents):
+ for idx, line in enumerate(contents):
+ if line[-1] == "\n":
+ line = line[:-1]
+ else:
+ report_error_name_no(name, idx + 1, "No newline at EOF")
+
+ if line.endswith(' '):
+ report_error_name_no(name, idx + 1, "trailing whitespace")
+
+ if '\t' in line:
+ report_error_name_no(name, idx + 1, "tab on line")
+
+ if '\r' in line:
+ report_error_name_no(name, idx + 1, "CR on line")
+
+
+exceptions = [
+ # Upstream
+ "support",
+ "tests/wpt/web-platform-tests",
+
+ # Generated and upstream code combined with our own. Could use cleanup
+ "components/script/dom/bindings/codegen",
+ "components/style/properties/mod.rs",
+]
+
+
+def should_check(name):
+ if ".#" in name:
+ return False
+ if not (name.endswith(".rs")
+ or name.endswith(".rc")
+ or name.endswith(".cpp")
+ or name.endswith(".c")
+ or name.endswith(".h")
+ or name.endswith(".py")):
+ return False
+ for exception in exceptions:
+ if exception in name:
+ return False
+ return True
+
+
+def scan(start_path):
+ global err
+ err = 0
+
+ file_names = []
+ for root, dirs, files in os.walk(start_path):
+ for myfile in files:
+ file_name = root + "/" + myfile
+ if should_check(file_name):
+ file_names.append(file_name)
+
+ for path in file_names:
+ with open(path, "r") as fp:
+ lines = fp.readlines()
+ do_license_check(path, "".join(lines))
+ do_whitespace_check(path, lines)
+
+ return err
diff --git a/python/toml/LICENSE b/python/toml/LICENSE
new file mode 100644
index 00000000000..c55b493cb7e
--- /dev/null
+++ b/python/toml/LICENSE
@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright 2013 Uiri Noyb
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE. \ No newline at end of file
diff --git a/python/toml/PKG-INFO b/python/toml/PKG-INFO
new file mode 100644
index 00000000000..c9c5493f311
--- /dev/null
+++ b/python/toml/PKG-INFO
@@ -0,0 +1,52 @@
+Metadata-Version: 1.0
+Name: toml
+Version: 0.8.2
+Summary: Python Library for Tom's Obvious, Minimal Language
+Home-page: https://github.com/uiri/toml
+Author: Uiri Noyb
+Author-email: uiri@xqz.ca
+License: License :: OSI Approved :: MIT License
+Description: TOML
+ ====
+
+ Original repository: https://github.com/uiri/toml
+
+ See also https://github.com/mojombo/toml
+
+ Python module which parses and emits TOML.
+
+ Released under the MIT license.
+
+ Passes https://github.com/BurntSushi/toml-test
+
+ See http://j.xqz.ca/toml-status for up to date test results.
+
+ Current Version of the Specification
+ ------------------------------------
+
+ https://github.com/mojombo/toml/blob/v0.2.0/README.md
+
+ QUICK GUIDE
+ -----------
+
+ ``pip install toml``
+
+ toml.loads --- takes a string to be parsed as toml and returns the corresponding dictionary
+
+ toml.dumps --- takes a dictionary and returns a string which is the contents of the corresponding toml file.
+
+
+ There are other functions which I use to dump and load various fragments of toml but dumps and loads will cover most usage.
+
+ Example usage:
+
+ .. code:: python
+
+ import toml
+
+ with open("conf.toml") as conffile:
+ config = toml.loads(conffile.read())
+ # do stuff with config here
+ . . .
+
+Platform: UNKNOWN
diff --git a/python/toml/README.rst b/python/toml/README.rst
new file mode 100644
index 00000000000..0c97ce21044
--- /dev/null
+++ b/python/toml/README.rst
@@ -0,0 +1,42 @@
+TOML
+====
+
+Original repository: https://github.com/uiri/toml
+
+See also https://github.com/mojombo/toml
+
+Python module which parses and emits TOML.
+
+Released under the MIT license.
+
+Passes https://github.com/BurntSushi/toml-test
+
+See http://j.xqz.ca/toml-status for up to date test results.
+
+Current Version of the Specification
+------------------------------------
+
+https://github.com/mojombo/toml/blob/v0.2.0/README.md
+
+QUICK GUIDE
+-----------
+
+``pip install toml``
+
+toml.loads --- takes a string to be parsed as toml and returns the corresponding dictionary
+
+toml.dumps --- takes a dictionary and returns a string which is the contents of the corresponding toml file.
+
+
+There are other functions which I use to dump and load various fragments of toml but dumps and loads will cover most usage.
+
+Example usage:
+
+.. code:: python
+
+ import toml
+
+ with open("conf.toml") as conffile:
+ config = toml.loads(conffile.read())
+ # do stuff with config here
+ . . .
diff --git a/python/toml/setup.py b/python/toml/setup.py
new file mode 100644
index 00000000000..e2f2492dc10
--- /dev/null
+++ b/python/toml/setup.py
@@ -0,0 +1,14 @@
+from distutils.core import setup
+
+with open("README.rst") as readmefile:
+ readme = readmefile.read()
+setup(name='toml',
+ version='0.8.2',
+ description="Python Library for Tom's Obvious, Minimal Language",
+ author="Uiri Noyb",
+ author_email="uiri@xqz.ca",
+ url="https://github.com/uiri/toml",
+ py_modules=['toml'],
+ license="License :: OSI Approved :: MIT License",
+ long_description=readme,
+)
diff --git a/python/toml/toml.py b/python/toml/toml.py
new file mode 100644
index 00000000000..8b3ecb67a2a
--- /dev/null
+++ b/python/toml/toml.py
@@ -0,0 +1,443 @@
+import datetime, decimal
+
+try:
+ _range = xrange
+except NameError:
+ unicode = str
+ _range = range
+ basestring = str
+ unichr = chr
+
+def load(f):
+ """Returns a dictionary containing the named file parsed as toml."""
+ if isinstance(f, basestring):
+ with open(f) as ffile:
+ return loads(ffile.read())
+ elif isinstance(f, list):
+ for l in f:
+ if not isinstance(l, basestring):
+ raise Exception("Load expects a list to contain filenames only")
+ d = []
+ for l in f:
+ d.append(load(l))
+ r = {}
+ for l in d:
+ toml_merge_dict(r, l)
+ return r
+ elif f.read:
+ return loads(f.read())
+ else:
+ raise Exception("You can only load a file descriptor, filename or list")
+
+def loads(s):
+ """Returns a dictionary containing s, a string, parsed as toml."""
+ implicitgroups = []
+ retval = {}
+ currentlevel = retval
+ if isinstance(s, basestring):
+ try:
+ s.decode('utf8')
+ except AttributeError:
+ pass
+ sl = list(s)
+ openarr = 0
+ openstring = False
+ arrayoftables = True
+ beginline = True
+ keygroup = False
+ delnum = 1
+ for i in range(len(sl)):
+ if sl[i] == '"':
+ oddbackslash = False
+ try:
+ k = 1
+ j = sl[i-k]
+ oddbackslash = False
+ while j == '\\':
+ oddbackslash = not oddbackslash
+ k += 1
+ j = sl[i-k]
+ except IndexError:
+ pass
+ if not oddbackslash:
+ openstring = not openstring
+ if keygroup and (sl[i] == ' ' or sl[i] == '\t'):
+ keygroup = False
+ if arrayoftables and (sl[i] == ' ' or sl[i] == '\t'):
+ arrayoftables = False
+ if sl[i] == '#' and not openstring and not keygroup and not arrayoftables:
+ j = i
+ while sl[j] != '\n':
+ sl.insert(j, ' ')
+ sl.pop(j+1)
+ j += 1
+ if sl[i] == '[' and not openstring and not keygroup and not arrayoftables:
+ if beginline:
+ if sl[i+1] == '[':
+ arrayoftables = True
+ else:
+ keygroup = True
+ else:
+ openarr += 1
+ if sl[i] == ']' and not openstring and not keygroup and not arrayoftables:
+ if keygroup:
+ keygroup = False
+ elif arrayoftables:
+ if sl[i-1] == ']':
+ arrayoftables = False
+ else:
+ openarr -= 1
+ if sl[i] == '\n':
+ if openstring:
+ raise Exception("Unbalanced quotes")
+ if openarr:
+ sl.insert(i, ' ')
+ sl.pop(i+1)
+ else:
+ beginline = True
+ elif beginline and sl[i] != ' ' and sl[i] != '\t':
+ beginline = False
+ keygroup = True
+ s = ''.join(sl)
+ s = s.split('\n')
+ else:
+ raise Exception("What exactly are you trying to pull?")
+ for line in s:
+ line = line.strip()
+ if line == "":
+ continue
+ if line[0] == '[':
+ arrayoftables = False
+ if line[1] == '[':
+ arrayoftables = True
+ line = line[2:].split(']]', 1)
+ else:
+ line = line[1:].split(']', 1)
+ if line[1].strip() != "":
+ raise Exception("Key group not on a line by itself.")
+ line = line[0]
+ if '[' in line:
+ raise Exception("Key group name cannot contain '['")
+ if ']' in line:
+ raise Exception("Key group name cannot contain']'")
+ groups = line.split('.')
+ currentlevel = retval
+ for i in range(len(groups)):
+ group = groups[i]
+ if group == "":
+ raise Exception("Can't have a keygroup with an empty name")
+ try:
+ currentlevel[group]
+ if i == len(groups) - 1:
+ if group in implicitgroups:
+ implicitgroups.remove(group)
+ if arrayoftables:
+ raise Exception("An implicitly defined table can't be an array")
+ elif arrayoftables:
+ currentlevel[group].append({})
+ else:
+ raise Exception("What? "+group+" already exists?"+str(currentlevel))
+ except TypeError:
+ if i != len(groups) - 1:
+ implicitgroups.append(group)
+ currentlevel = currentlevel[0]
+ if arrayoftables:
+ currentlevel[group] = [{}]
+ else:
+ currentlevel[group] = {}
+ except KeyError:
+ if i != len(groups) - 1:
+ implicitgroups.append(group)
+ currentlevel[group] = {}
+ if i == len(groups) - 1 and arrayoftables:
+ currentlevel[group] = [{}]
+ currentlevel = currentlevel[group]
+ if arrayoftables:
+ try:
+ currentlevel = currentlevel[-1]
+ except KeyError:
+ pass
+ elif "=" in line:
+ i = 1
+ pair = line.split('=', i)
+ l = len(line)
+ while pair[-1][0] != ' ' and pair[-1][0] != '\t' and pair[-1][0] != '"' and pair[-1][0] != '[' and pair[-1] != 'true' and pair[-1] != 'false':
+ try:
+ float(pair[-1])
+ break
+ except ValueError:
+ try:
+ datetime.datetime.strptime(pair[-1], "%Y-%m-%dT%H:%M:%SZ")
+ break
+ except ValueError:
+ i += 1
+ pair = line.split('=', i)
+ newpair = []
+ newpair.append('='.join(pair[:-1]))
+ newpair.append(pair[-1])
+ pair = newpair
+ pair[0] = pair[0].strip()
+ pair[1] = pair[1].strip()
+ value, vtype = load_value(pair[1])
+ try:
+ currentlevel[pair[0]]
+ raise Exception("Duplicate keys!")
+ except KeyError:
+ currentlevel[pair[0]] = value
+ return retval
+
+def load_value(v):
+ if v == 'true':
+ return (True, "bool")
+ elif v == 'false':
+ return (False, "bool")
+ elif v[0] == '"':
+ testv = v[1:].split('"')
+ closed = False
+ for tv in testv:
+ if tv == '':
+ closed = True
+ else:
+ oddbackslash = False
+ try:
+ i = -1
+ j = tv[i]
+ while j == '\\':
+ oddbackslash = not oddbackslash
+ i -= 1
+ j = tv[i]
+ except IndexError:
+ pass
+ if not oddbackslash:
+ if closed:
+ raise Exception("Stuff after closed string. WTF?")
+ else:
+ closed = True
+ escapes = ['0', 'b', 'f', '/', 'n', 'r', 't', '"', '\\']
+ escapedchars = ['\0', '\b', '\f', '/', '\n', '\r', '\t', '\"', '\\']
+ escapeseqs = v.split('\\')[1:]
+ backslash = False
+ for i in escapeseqs:
+ if i == '':
+ backslash = not backslash
+ else:
+ if i[0] not in escapes and i[0] != 'u' and not backslash:
+ raise Exception("Reserved escape sequence used")
+ if backslash:
+ backslash = False
+ if "\\u" in v:
+ hexchars = ['0', '1', '2', '3', '4', '5', '6', '7',
+ '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']
+ hexbytes = v.split('\\u')
+ newv = hexbytes[0]
+ hexbytes = hexbytes[1:]
+ for hx in hexbytes:
+ hxb = ""
+ try:
+ if hx[0].lower() in hexchars:
+ hxb += hx[0].lower()
+ if hx[1].lower() in hexchars:
+ hxb += hx[1].lower()
+ if hx[2].lower() in hexchars:
+ hxb += hx[2].lower()
+ if hx[3].lower() in hexchars:
+ hxb += hx[3].lower()
+ except IndexError:
+ if len(hxb) != 2:
+ raise Exception("Invalid escape sequence")
+ if len(hxb) != 4 and len(hxb) != 2:
+ raise Exception("Invalid escape sequence")
+ newv += unichr(int(hxb, 16))
+ newv += unicode(hx[len(hxb):])
+ v = newv
+ for i in range(len(escapes)):
+ v = v.replace("\\"+escapes[i], escapedchars[i])
+ # (where (n) signifies a member of escapes:
+ # undo (\\)(\\)(n) -> (\\)(\n)
+ v = v.replace("\\"+escapedchars[i], "\\\\"+escapes[i])
+ return (v[1:-1], "str")
+ elif v[0] == '[':
+ return (load_array(v), "array")
+ elif len(v) == 20 and v[-1] == 'Z':
+ if v[10] == 'T':
+ return (datetime.datetime.strptime(v, "%Y-%m-%dT%H:%M:%SZ"), "date")
+ else:
+ raise Exception("Wait, what?")
+ else:
+ itype = "int"
+ digits = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
+ neg = False
+ if v[0] == '-':
+ neg = True
+ v = v[1:]
+ if '.' in v:
+ if v.split('.', 1)[1] == '':
+ raise Exception("This float is missing digits after the point")
+ if v[0] not in digits:
+ raise Exception("This float doesn't have a leading digit")
+ v = float(v)
+ itype = "float"
+ else:
+ v = int(v)
+ if neg:
+ return (0 - v, itype)
+ return (v, itype)
+
+
+def load_array(a):
+ atype = None
+ retval = []
+ a = a.strip()
+ if '[' not in a[1:-1]:
+ strarray = False
+ tmpa = a[1:-1].strip()
+ if tmpa != '' and tmpa[0] == '"':
+ strarray = True
+ a = a[1:-1].split(',')
+ b = 0
+ if strarray:
+ while b < len(a) - 1:
+ while a[b].strip()[-1] != '"' and a[b+1].strip()[0] != '"':
+ a[b] = a[b] + ',' + a[b+1]
+ if b < len(a) - 2:
+ a = a[:b+1] + a[b+2:]
+ else:
+ a = a[:b+1]
+ b += 1
+ else:
+ al = list(a[1:-1])
+ a = []
+ openarr = 0
+ j = 0
+ for i in range(len(al)):
+ if al[i] == '[':
+ openarr += 1
+ elif al[i] == ']':
+ openarr -= 1
+ elif al[i] == ',' and not openarr:
+ a.append(''.join(al[j:i]))
+ j = i+1
+ a.append(''.join(al[j:]))
+ for i in range(len(a)):
+ a[i] = a[i].strip()
+ if a[i] != '':
+ nval, ntype = load_value(a[i])
+ if atype:
+ if ntype != atype:
+ raise Exception("Not a homogeneous array")
+ else:
+ atype = ntype
+ retval.append(nval)
+ return retval
+
+def dump(o, f):
+ """Writes out to f the toml corresponding to o. Returns said toml."""
+ if f.write:
+ d = dumps(o)
+ f.write(d)
+ return d
+ else:
+ raise Exception("You can only dump an object to a file descriptor")
+
+def dumps(o):
+ """Returns a string containing the toml corresponding to o, a dictionary"""
+ retval = ""
+ addtoretval, sections = dump_sections(o, "")
+ retval += addtoretval
+ while sections != {}:
+ newsections = {}
+ for section in sections:
+ addtoretval, addtosections = dump_sections(sections[section], section)
+ if addtoretval:
+ retval += "["+section+"]\n"
+ retval += addtoretval
+ for s in addtosections:
+ newsections[section+"."+s] = addtosections[s]
+ sections = newsections
+ return retval
+
+def dump_sections(o, sup):
+ retstr = ""
+ if sup != "" and sup[-1] != ".":
+ sup += '.'
+ retdict = {}
+ arraystr = ""
+ for section in o:
+ if not isinstance(o[section], dict):
+ arrayoftables = False
+ if isinstance(o[section], list):
+ for a in o[section]:
+ if isinstance(a, dict):
+ arrayoftables = True
+ if arrayoftables:
+ for a in o[section]:
+ arraytabstr = ""
+ arraystr += "[["+sup+section+"]]\n"
+ s, d = dump_sections(a, sup+section)
+ if s:
+ if s[0] == "[":
+ arraytabstr += s
+ else:
+ arraystr += s
+ while d != {}:
+ newd = {}
+ for dsec in d:
+ s1, d1 = dump_sections(d[dsec], sup+section+dsec)
+ if s1:
+ arraytabstr += "["+sup+section+"."+dsec+"]\n"
+ arraytabstr += s1
+ for s1 in d1:
+ newd[dsec+"."+s1] = d1[s1]
+ d = newd
+ arraystr += arraytabstr
+ else:
+ retstr += section + " = " + str(dump_value(o[section])) + '\n'
+ else:
+ retdict[section] = o[section]
+ retstr += arraystr
+ return (retstr, retdict)
+
+def dump_value(v):
+ if isinstance(v, list):
+ t = []
+ retval = "["
+ for u in v:
+ t.append(dump_value(u))
+ while t != []:
+ s = []
+ for u in t:
+ if isinstance(u, list):
+ for r in u:
+ s.append(r)
+ else:
+ retval += " " + str(u) + ","
+ t = s
+ retval += "]"
+ return retval
+ if isinstance(v, (str, unicode)):
+ escapes = ['\\', '0', 'b', 'f', '/', 'n', 'r', 't', '"']
+ escapedchars = ['\\', '\0', '\b', '\f', '/', '\n', '\r', '\t', '\"']
+ for i in range(len(escapes)):
+ v = v.replace(escapedchars[i], "\\"+escapes[i])
+ return str('"'+v+'"')
+ if isinstance(v, bool):
+ return str(v).lower()
+ if isinstance(v, datetime.datetime):
+ return v.isoformat()[:19]+'Z'
+ if isinstance(v, float):
+ return '{0:f}'.format(decimal.Decimal(str(v)))
+ return v
+
+def toml_merge_dict(a, b):
+ for k in a:
+ if isinstance(a[k], dict):
+ try:
+ b[k]
+ except KeyError:
+ continue
+ if isinstance(b[k], dict):
+ b[k] = toml_merge_dict(a[k], b[k])
+ else:
+ raise Exception("Can't merge dict and nondict in toml object")
+ a.update(b)
+ return a