diff options
Diffstat (limited to 'python')
37 files changed, 1 insertions, 3793 deletions
diff --git a/python/mach/README.rst b/python/mach/README.rst deleted file mode 100644 index 7c2e00becba..00000000000 --- a/python/mach/README.rst +++ /dev/null @@ -1,13 +0,0 @@ -==== -mach -==== - -Mach (German for *do*) is a generic command dispatcher for the command -line. - -To use mach, you install the mach core (a Python package), create an -executable *driver* script (named whatever you want), and write mach -commands. When the *driver* is executed, mach dispatches to the -requested command handler automatically. - -To learn more, read the docs in ``docs/``. diff --git a/python/mach/bash-completion.sh b/python/mach/bash-completion.sh deleted file mode 100644 index e4b151f24c9..00000000000 --- a/python/mach/bash-completion.sh +++ /dev/null @@ -1,29 +0,0 @@ -function _mach() -{ - local cur cmds c subcommand - COMPREPLY=() - - # Load the list of commands - cmds=`"${COMP_WORDS[0]}" mach-commands` - - # Look for the subcommand. - cur="${COMP_WORDS[COMP_CWORD]}" - subcommand="" - c=1 - while [ $c -lt $COMP_CWORD ]; do - word="${COMP_WORDS[c]}" - for cmd in $cmds; do - if [ "$cmd" = "$word" ]; then - subcommand="$word" - fi - done - c=$((++c)) - done - - if [[ "$subcommand" == "help" || -z "$subcommand" ]]; then - COMPREPLY=( $(compgen -W "$cmds" -- ${cur}) ) - fi - - return 0 -} -complete -o default -F _mach mach diff --git a/python/mach/docs/commands.rst b/python/mach/docs/commands.rst deleted file mode 100644 index af2973dd7e7..00000000000 --- a/python/mach/docs/commands.rst +++ /dev/null @@ -1,145 +0,0 @@ -.. _mach_commands: - -===================== -Implementing Commands -===================== - -Mach commands are defined via Python decorators. - -All the relevant decorators are defined in the *mach.decorators* module. -The important decorators are as follows: - -:py:func:`CommandProvider <mach.decorators.CommandProvider>` - A class decorator that denotes that a class contains mach - commands. The decorator takes no arguments. - -:py:func:`Command <mach.decorators.Command>` - A method decorator that denotes that the method should be called when - the specified command is requested. The decorator takes a command name - as its first argument and a number of additional arguments to - configure the behavior of the command. - -:py:func:`CommandArgument <mach.decorators.CommandArgument>` - A method decorator that defines an argument to the command. Its - arguments are essentially proxied to ArgumentParser.add_argument() - -:py:func:`SubCommand <mach.decorators.SubCommand>` - A method decorator that denotes that the method should be a - sub-command to an existing ``@Command``. The decorator takes the - parent command name as its first argument and the sub-command name - as its second argument. - - ``@CommandArgument`` can be used on ``@SubCommand`` instances just - like they can on ``@Command`` instances. - -Classes with the ``@CommandProvider`` decorator **must** have an -``__init__`` method that accepts 1 or 2 arguments. If it accepts 2 -arguments, the 2nd argument will be a -:py:class:`mach.base.CommandContext` instance. - -Here is a complete example: - -.. code-block:: python - - from mach.decorators import ( - CommandArgument, - CommandProvider, - Command, - ) - - @CommandProvider - class MyClass(object): - @Command('doit', help='Do ALL OF THE THINGS.') - @CommandArgument('--force', '-f', action='store_true', - help='Force doing it.') - def doit(self, force=False): - # Do stuff here. - -When the module is loaded, the decorators tell mach about all handlers. -When mach runs, it takes the assembled metadata from these handlers and -hooks it up to the command line driver. Under the hood, arguments passed -to the decorators are being used to help mach parse command arguments, -formulate arguments to the methods, etc. See the documentation in the -:py:mod:`mach.base` module for more. - -The Python modules defining mach commands do not need to live inside the -main mach source tree. - -Conditionally Filtering Commands -================================ - -Sometimes it might only make sense to run a command given a certain -context. For example, running tests only makes sense if the product -they are testing has been built, and said build is available. To make -sure a command is only runnable from within a correct context, you can -define a series of conditions on the -:py:func:`Command <mach.decorators.Command>` decorator. - -A condition is simply a function that takes an instance of the -:py:func:`mach.decorators.CommandProvider` class as an argument, and -returns ``True`` or ``False``. If any of the conditions defined on a -command return ``False``, the command will not be runnable. The -docstring of a condition function is used in error messages, to explain -why the command cannot currently be run. - -Here is an example: - -.. code-block:: python - - from mach.decorators import ( - CommandProvider, - Command, - ) - - def build_available(cls): - """The build needs to be available.""" - return cls.build_path is not None - - @CommandProvider - class MyClass(MachCommandBase): - def __init__(self, build_path=None): - self.build_path = build_path - - @Command('run_tests', conditions=[build_available]) - def run_tests(self): - # Do stuff here. - -It is important to make sure that any state needed by the condition is -available to instances of the command provider. - -By default all commands without any conditions applied will be runnable, -but it is possible to change this behaviour by setting -``require_conditions`` to ``True``: - -.. code-block:: python - - m = mach.main.Mach() - m.require_conditions = True - -Minimizing Code in Commands -=========================== - -Mach command modules, classes, and methods work best when they are -minimal dispatchers. The reason is import bloat. Currently, the mach -core needs to import every Python file potentially containing mach -commands for every command invocation. If you have dozens of commands or -commands in modules that import a lot of Python code, these imports -could slow mach down and waste memory. - -It is thus recommended that mach modules, classes, and methods do as -little work as possible. Ideally the module should only import from -the :py:mod:`mach` package. If you need external modules, you should -import them from within the command method. - -To keep code size small, the body of a command method should be limited -to: - -1. Obtaining user input (parsing arguments, prompting, etc) -2. Calling into some other Python package -3. Formatting output - -Of course, these recommendations can be ignored if you want to risk -slower performance. - -In the future, the mach driver may cache the dispatching information or -have it intelligently loaded to facilitate lazy loading. diff --git a/python/mach/docs/driver.rst b/python/mach/docs/driver.rst deleted file mode 100644 index 022ebe65739..00000000000 --- a/python/mach/docs/driver.rst +++ /dev/null @@ -1,51 +0,0 @@ -.. _mach_driver: - -======= -Drivers -======= - -Entry Points -============ - -It is possible to use setuptools' entry points to load commands -directly from python packages. A mach entry point is a function which -returns a list of files or directories containing mach command -providers. e.g.: - -.. code-block:: python - - def list_providers(): - providers = [] - here = os.path.abspath(os.path.dirname(__file__)) - for p in os.listdir(here): - if p.endswith('.py'): - providers.append(os.path.join(here, p)) - return providers - -See http://pythonhosted.org/setuptools/setuptools.html#dynamic-discovery-of-services-and-plugins -for more information on creating an entry point. To search for entry -point plugins, you can call -:py:meth:`mach.main.Mach.load_commands_from_entry_point`. e.g.: - -.. code-block:: python - - mach.load_commands_from_entry_point("mach.external.providers") - -Adding Global Arguments -======================= - -Arguments to mach commands are usually command-specific. However, -mach ships with a handful of global arguments that apply to all -commands. - -It is possible to extend the list of global arguments. In your -*mach driver*, simply call -:py:meth:`mach.main.Mach.add_global_argument`. e.g.: - -.. code-block:: python - - mach = mach.main.Mach(os.getcwd()) - - # Will allow --example to be specified on every mach command. - mach.add_global_argument('--example', action='store_true', - help='Demonstrate an example global argument.') diff --git a/python/mach/docs/index.rst b/python/mach/docs/index.rst deleted file mode 100644 index b4213cb7834..00000000000 --- a/python/mach/docs/index.rst +++ /dev/null @@ -1,74 +0,0 @@ -==== -mach -==== - -Mach (German for *do*) is a generic command dispatcher for the command -line. - -To use mach, you install the mach core (a Python package), create an -executable *driver* script (named whatever you want), and write mach -commands. When the *driver* is executed, mach dispatches to the -requested command handler automatically. - -Features -======== - -On a high level, mach is similar to using argparse with subparsers (for -command handling). When you dig deeper, mach offers a number of -additional features: - -Distributed command definitions - With optparse/argparse, you have to define your commands on a central - parser instance. With mach, you annotate your command methods with - decorators and mach finds and dispatches to them automatically. - -Command categories - Mach commands can be grouped into categories when displayed in help. - This is currently not possible with argparse. - -Logging management - Mach provides a facility for logging (both classical text and - structured) that is available to any command handler. - -Settings files - Mach provides a facility for reading settings from an ini-like file - format. - -Components -========== - -Mach is conceptually composed of the following components: - -core - The mach core is the core code powering mach. This is a Python package - that contains all the business logic that makes mach work. The mach - core is common to all mach deployments. - -commands - These are what mach dispatches to. Commands are simply Python methods - registered as command names. The set of commands is unique to the - environment mach is deployed in. - -driver - The *driver* is the entry-point to mach. It is simply an executable - script that loads the mach core, tells it where commands can be found, - then asks the mach core to handle the current request. The driver is - unique to the deployed environment. But, it's usually based on an - example from this source tree. - -Project State -============= - -mach was originally written as a command dispatching framework to aid -Firefox development. While the code is mostly generic, there are still -some pieces that closely tie it to Mozilla/Firefox. The goal is for -these to eventually be removed and replaced with generic features so -mach is suitable for anybody to use. Until then, mach may not be the -best fit for you. - -.. toctree:: - :maxdepth: 1 - - commands - driver - logging diff --git a/python/mach/docs/logging.rst b/python/mach/docs/logging.rst deleted file mode 100644 index ff245cf0320..00000000000 --- a/python/mach/docs/logging.rst +++ /dev/null @@ -1,100 +0,0 @@ -.. _mach_logging: - -======= -Logging -======= - -Mach configures a built-in logging facility so commands can easily log -data. - -What sets the logging facility apart from most loggers you've seen is -that it encourages structured logging. Instead of conventional logging -where simple strings are logged, the internal logging mechanism logs all -events with the following pieces of information: - -* A string *action* -* A dict of log message fields -* A formatting string - -Essentially, instead of assembling a human-readable string at -logging-time, you create an object holding all the pieces of data that -will constitute your logged event. For each unique type of logged event, -you assign an *action* name. - -Depending on how logging is configured, your logged event could get -written a couple of different ways. - -JSON Logging -============ - -Where machines are the intended target of the logging data, a JSON -logger is configured. The JSON logger assembles an array consisting of -the following elements: - -* Decimal wall clock time in seconds since UNIX epoch -* String *action* of message -* Object with structured message data - -The JSON-serialized array is written to a configured file handle. -Consumers of this logging stream can just perform a readline() then feed -that into a JSON deserializer to reconstruct the original logged -message. They can key off the *action* element to determine how to -process individual events. There is no need to invent a parser. -Convenient, isn't it? - -Logging for Humans -================== - -Where humans are the intended consumer of a log message, the structured -log message are converted to more human-friendly form. This is done by -utilizing the *formatting* string provided at log time. The logger -simply calls the *format* method of the formatting string, passing the -dict containing the message's fields. - -When *mach* is used in a terminal that supports it, the logging facility -also supports terminal features such as colorization. This is done -automatically in the logging layer - there is no need to control this at -logging time. - -In addition, messages intended for humans typically prepends every line -with the time passed since the application started. - -Logging HOWTO -============= - -Structured logging piggybacks on top of Python's built-in logging -infrastructure provided by the *logging* package. We accomplish this by -taking advantage of *logging.Logger.log()*'s *extra* argument. To this -argument, we pass a dict with the fields *action* and *params*. These -are the string *action* and dict of message fields, respectively. The -formatting string is passed as the *msg* argument, like normal. - -If you were logging to a logger directly, you would do something like: - -.. code-block:: python - - logger.log(logging.INFO, 'My name is {name}', - extra={'action': 'my_name', 'params': {'name': 'Gregory'}}) - -The JSON logging would produce something like:: - - [1339985554.306338, "my_name", {"name": "Gregory"}] - -Human logging would produce something like:: - - 0.52 My name is Gregory - -Since there is a lot of complexity using logger.log directly, it is -recommended to go through a wrapping layer that hides part of the -complexity for you. The easiest way to do this is by utilizing the -LoggingMixin: - -.. code-block:: python - - import logging - from mach.mixin.logging import LoggingMixin - - class MyClass(LoggingMixin): - def foo(self): - self.log(logging.INFO, 'foo_start', {'bar': True}, - 'Foo performed. Bar: {bar}') diff --git a/python/mach/mach/__init__.py b/python/mach/mach/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/python/mach/mach/__init__.py +++ /dev/null diff --git a/python/mach/mach/base.py b/python/mach/mach/base.py deleted file mode 100644 index 3556dc6e577..00000000000 --- a/python/mach/mach/base.py +++ /dev/null @@ -1,46 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -from __future__ import absolute_import, unicode_literals - - -class CommandContext(object): - """Holds run-time state so it can easily be passed to command providers.""" - def __init__(self, cwd=None, settings=None, log_manager=None, - commands=None, **kwargs): - self.cwd = cwd - self.settings = settings - self.log_manager = log_manager - self.commands = commands - - for k,v in kwargs.items(): - setattr(self, k, v) - - -class MachError(Exception): - """Base class for all errors raised by mach itself.""" - - -class NoCommandError(MachError): - """No command was passed into mach.""" - - -class UnknownCommandError(MachError): - """Raised when we attempted to execute an unknown command.""" - - def __init__(self, command, verb, suggested_commands=None): - MachError.__init__(self) - - self.command = command - self.verb = verb - self.suggested_commands = suggested_commands or [] - -class UnrecognizedArgumentError(MachError): - """Raised when an unknown argument is passed to mach.""" - - def __init__(self, command, arguments): - MachError.__init__(self) - - self.command = command - self.arguments = arguments diff --git a/python/mach/mach/commands/__init__.py b/python/mach/mach/commands/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/python/mach/mach/commands/__init__.py +++ /dev/null diff --git a/python/mach/mach/commands/commandinfo.py b/python/mach/mach/commands/commandinfo.py deleted file mode 100644 index e93bdd58e29..00000000000 --- a/python/mach/mach/commands/commandinfo.py +++ /dev/null @@ -1,47 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, # You can obtain one at http://mozilla.org/MPL/2.0/. - -from __future__ import absolute_import, print_function, unicode_literals - -from mach.decorators import ( - CommandProvider, - Command, - CommandArgument, -) - - -@CommandProvider -class BuiltinCommands(object): - def __init__(self, context): - self.context = context - - @Command('mach-commands', category='misc', - description='List all mach commands.') - def commands(self): - print("\n".join(self.context.commands.command_handlers.keys())) - - @Command('mach-debug-commands', category='misc', - description='Show info about available mach commands.') - @CommandArgument('match', metavar='MATCH', default=None, nargs='?', - help='Only display commands containing given substring.') - def debug_commands(self, match=None): - import inspect - - handlers = self.context.commands.command_handlers - for command in sorted(handlers.keys()): - if match and match not in command: - continue - - handler = handlers[command] - cls = handler.cls - method = getattr(cls, getattr(handler, 'method')) - - print(command) - print('=' * len(command)) - print('') - print('File: %s' % inspect.getsourcefile(method)) - print('Class: %s' % cls.__name__) - print('Method: %s' % handler.method) - print('') - diff --git a/python/mach/mach/commands/settings.py b/python/mach/mach/commands/settings.py deleted file mode 100644 index 14c08928a13..00000000000 --- a/python/mach/mach/commands/settings.py +++ /dev/null @@ -1,50 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at http://mozilla.org/MPL/2.0/. - -from __future__ import absolute_import, print_function, unicode_literals - -from textwrap import TextWrapper - -from mach.decorators import ( - CommandProvider, - Command, -) - - -#@CommandProvider -class Settings(object): - """Interact with settings for mach. - - Currently, we only provide functionality to view what settings are - available. In the future, this module will be used to modify settings, help - people create configs via a wizard, etc. - """ - def __init__(self, context): - self.settings = context.settings - - @Command('settings-list', category='devenv', - description='Show available config settings.') - def list_settings(self): - """List available settings in a concise list.""" - for section in sorted(self.settings): - for option in sorted(self.settings[section]): - short, full = self.settings.option_help(section, option) - print('%s.%s -- %s' % (section, option, short)) - - @Command('settings-create', category='devenv', - description='Print a new settings file with usage info.') - def create(self): - """Create an empty settings file with full documentation.""" - wrapper = TextWrapper(initial_indent='# ', subsequent_indent='# ') - - for section in sorted(self.settings): - print('[%s]' % section) - print('') - - for option in sorted(self.settings[section]): - short, full = self.settings.option_help(section, option) - - print(wrapper.fill(full)) - print(';%s =' % option) - print('') diff --git a/python/mach/mach/config.py b/python/mach/mach/config.py deleted file mode 100644 index 5864e5e6a28..00000000000 --- a/python/mach/mach/config.py +++ /dev/null @@ -1,488 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at http://mozilla.org/MPL/2.0/. - -r""" -This file defines classes for representing config data/settings. - -Config data is modeled as key-value pairs. Keys are grouped together into named -sections. Individual config settings (options) have metadata associated with -them. This metadata includes type, default value, valid values, etc. - -The main interface to config data is the ConfigSettings class. 1 or more -ConfigProvider classes are associated with ConfigSettings and define what -settings are available. - -Descriptions of individual config options can be translated to multiple -languages using gettext. Each option has associated with it a domain and locale -directory. By default, the domain is the section the option is in and the -locale directory is the "locale" directory beneath the directory containing the -module that defines it. - -People implementing ConfigProvider instances are expected to define a complete -gettext .po and .mo file for the en-US locale. You can use the gettext-provided -msgfmt binary to perform this conversion. Generation of the original .po file -can be done via the write_pot() of ConfigSettings. -""" - -from __future__ import absolute_import, unicode_literals - -import collections -import gettext -import os -import sys - -if sys.version_info[0] == 3: - from configparser import RawConfigParser - str_type = str -else: - from ConfigParser import RawConfigParser - str_type = basestring - - -class ConfigType(object): - """Abstract base class for config values.""" - - @staticmethod - def validate(value): - """Validates a Python value conforms to this type. - - Raises a TypeError or ValueError if it doesn't conform. Does not do - anything if the value is valid. - """ - - @staticmethod - def from_config(config, section, option): - """Obtain the value of this type from a RawConfigParser. - - Receives a RawConfigParser instance, a str section name, and the str - option in that section to retrieve. - - The implementation may assume the option exists in the RawConfigParser - instance. - - Implementations are not expected to validate the value. But, they - should return the appropriate Python type. - """ - - @staticmethod - def to_config(value): - return value - - -class StringType(ConfigType): - @staticmethod - def validate(value): - if not isinstance(value, str_type): - raise TypeError() - - @staticmethod - def from_config(config, section, option): - return config.get(section, option) - - -class BooleanType(ConfigType): - @staticmethod - def validate(value): - if not isinstance(value, bool): - raise TypeError() - - @staticmethod - def from_config(config, section, option): - return config.getboolean(section, option) - - @staticmethod - def to_config(value): - return 'true' if value else 'false' - - -class IntegerType(ConfigType): - @staticmethod - def validate(value): - if not isinstance(value, int): - raise TypeError() - - @staticmethod - def from_config(config, section, option): - return config.getint(section, option) - - -class PositiveIntegerType(IntegerType): - @staticmethod - def validate(value): - if not isinstance(value, int): - raise TypeError() - - if value < 0: - raise ValueError() - - -class PathType(StringType): - @staticmethod - def validate(value): - if not isinstance(value, str_type): - raise TypeError() - - @staticmethod - def from_config(config, section, option): - return config.get(section, option) - - -class AbsolutePathType(PathType): - @staticmethod - def validate(value): - if not isinstance(value, str_type): - raise TypeError() - - if not os.path.isabs(value): - raise ValueError() - - -class RelativePathType(PathType): - @staticmethod - def validate(value): - if not isinstance(value, str_type): - raise TypeError() - - if os.path.isabs(value): - raise ValueError() - - -class DefaultValue(object): - pass - - -class ConfigProvider(object): - """Abstract base class for an object providing config settings. - - Classes implementing this interface expose configurable settings. Settings - are typically only relevant to that component itself. But, nothing says - settings can't be shared by multiple components. - """ - - @classmethod - def register_settings(cls): - """Registers config settings. - - This is called automatically. Child classes should likely not touch it. - See _register_settings() instead. - """ - if hasattr(cls, '_settings_registered'): - return - - cls._settings_registered = True - - cls.config_settings = {} - - ourdir = os.path.dirname(__file__) - cls.config_settings_locale_directory = os.path.join(ourdir, 'locale') - - cls._register_settings() - - @classmethod - def _register_settings(cls): - """The actual implementation of register_settings(). - - This is what child classes should implement. They should not touch - register_settings(). - - Implementations typically make 1 or more calls to _register_setting(). - """ - raise NotImplemented('%s must implement _register_settings.' % - __name__) - - @classmethod - def register_setting(cls, section, option, type_cls, default=DefaultValue, - choices=None, domain=None): - """Register a config setting with this type. - - This is a convenience method to populate available settings. It is - typically called in the class's _register_settings() implementation. - - Each setting must have: - - section -- str section to which the setting belongs. This is how - settings are grouped. - - option -- str id for the setting. This must be unique within the - section it appears. - - type -- a ConfigType-derived type defining the type of the setting. - - Each setting has the following optional parameters: - - default -- The default value for the setting. If None (the default) - there is no default. - - choices -- A set of values this setting can hold. Values not in - this set are invalid. - - domain -- Translation domain for this setting. By default, the - domain is the same as the section name. - """ - if not section in cls.config_settings: - cls.config_settings[section] = {} - - if option in cls.config_settings[section]: - raise Exception('Setting has already been registered: %s.%s' % ( - section, option)) - - domain = domain if domain is not None else section - - meta = { - 'short': '%s.short' % option, - 'full': '%s.full' % option, - 'type_cls': type_cls, - 'domain': domain, - 'localedir': cls.config_settings_locale_directory, - } - - if default != DefaultValue: - meta['default'] = default - - if choices is not None: - meta['choices'] = choices - - cls.config_settings[section][option] = meta - - -class ConfigSettings(collections.Mapping): - """Interface for configuration settings. - - This is the main interface to the configuration. - - A configuration is a collection of sections. Each section contains - key-value pairs. - - When an instance is created, the caller first registers ConfigProvider - instances with it. This tells the ConfigSettings what individual settings - are available and defines extra metadata associated with those settings. - This is used for validation, etc. - - Once ConfigProvider instances are registered, a config is populated. It can - be loaded from files or populated by hand. - - ConfigSettings instances are accessed like dictionaries or by using - attributes. e.g. the section "foo" is accessed through either - settings.foo or settings['foo']. - - Sections are modeled by the ConfigSection class which is defined inside - this one. They look just like dicts or classes with attributes. To access - the "bar" option in the "foo" section: - - value = settings.foo.bar - value = settings['foo']['bar'] - value = settings.foo['bar'] - - Assignment is similar: - - settings.foo.bar = value - settings['foo']['bar'] = value - settings['foo'].bar = value - - You can even delete user-assigned values: - - del settings.foo.bar - del settings['foo']['bar'] - - If there is a default, it will be returned. - - When settings are mutated, they are validated against the registered - providers. Setting unknown settings or setting values to illegal values - will result in exceptions being raised. - """ - - class ConfigSection(collections.MutableMapping, object): - """Represents an individual config section.""" - def __init__(self, config, name, settings): - object.__setattr__(self, '_config', config) - object.__setattr__(self, '_name', name) - object.__setattr__(self, '_settings', settings) - - # MutableMapping interface - def __len__(self): - return len(self._settings) - - def __iter__(self): - return iter(self._settings.keys()) - - def __contains__(self, k): - return k in self._settings - - def __getitem__(self, k): - if k not in self._settings: - raise KeyError('Option not registered with provider: %s' % k) - - meta = self._settings[k] - - if self._config.has_option(self._name, k): - return meta['type_cls'].from_config(self._config, self._name, k) - - if not 'default' in meta: - raise KeyError('No default value registered: %s' % k) - - return meta['default'] - - def __setitem__(self, k, v): - if k not in self._settings: - raise KeyError('Option not registered with provider: %s' % k) - - meta = self._settings[k] - - meta['type_cls'].validate(v) - - if not self._config.has_section(self._name): - self._config.add_section(self._name) - - self._config.set(self._name, k, meta['type_cls'].to_config(v)) - - def __delitem__(self, k): - self._config.remove_option(self._name, k) - - # Prune empty sections. - if not len(self._config.options(self._name)): - self._config.remove_section(self._name) - - def __getattr__(self, k): - return self.__getitem__(k) - - def __setattr__(self, k, v): - self.__setitem__(k, v) - - def __delattr__(self, k): - self.__delitem__(k) - - - def __init__(self): - self._config = RawConfigParser() - - self._settings = {} - self._sections = {} - self._finalized = False - self._loaded_filenames = set() - - def load_file(self, filename): - self.load_files([filename]) - - def load_files(self, filenames): - """Load a config from files specified by their paths. - - Files are loaded in the order given. Subsequent files will overwrite - values from previous files. If a file does not exist, it will be - ignored. - """ - filtered = [f for f in filenames if os.path.exists(f)] - - fps = [open(f, 'rt') for f in filtered] - self.load_fps(fps) - self._loaded_filenames.update(set(filtered)) - for fp in fps: - fp.close() - - def load_fps(self, fps): - """Load config data by reading file objects.""" - - for fp in fps: - self._config.readfp(fp) - - def loaded_files(self): - return self._loaded_filenames - - def write(self, fh): - """Write the config to a file object.""" - self._config.write(fh) - - def validate(self): - """Ensure that the current config passes validation. - - This is a generator of tuples describing any validation errors. The - elements of the tuple are: - - (bool) True if error is fatal. False if just a warning. - (str) Type of validation issue. Can be one of ('unknown-section', - 'missing-required', 'type-error') - """ - - def register_provider(self, provider): - """Register a ConfigProvider with this settings interface.""" - - if self._finalized: - raise Exception('Providers cannot be registered after finalized.') - - provider.register_settings() - - for section_name, settings in provider.config_settings.items(): - section = self._settings.get(section_name, {}) - - for k, v in settings.items(): - if k in section: - raise Exception('Setting already registered: %s.%s' % - section_name, k) - - section[k] = v - - self._settings[section_name] = section - - def write_pot(self, fh): - """Write a pot gettext translation file.""" - - for section in sorted(self): - fh.write('# Section %s\n\n' % section) - for option in sorted(self[section]): - fh.write('msgid "%s.%s.short"\n' % (section, option)) - fh.write('msgstr ""\n\n') - - fh.write('msgid "%s.%s.full"\n' % (section, option)) - fh.write('msgstr ""\n\n') - - fh.write('# End of section %s\n\n' % section) - - def option_help(self, section, option): - """Obtain the translated help messages for an option.""" - - meta = self[section]._settings[option] - - # Providers should always have an en-US translation. If they don't, - # they are coded wrong and this will raise. - default = gettext.translation(meta['domain'], meta['localedir'], - ['en-US']) - - t = gettext.translation(meta['domain'], meta['localedir'], - fallback=True) - t.add_fallback(default) - - short = t.ugettext('%s.%s.short' % (section, option)) - full = t.ugettext('%s.%s.full' % (section, option)) - - return (short, full) - - def _finalize(self): - if self._finalized: - return - - for section, settings in self._settings.items(): - s = ConfigSettings.ConfigSection(self._config, section, settings) - self._sections[section] = s - - self._finalized = True - - # Mapping interface. - def __len__(self): - return len(self._settings) - - def __iter__(self): - self._finalize() - - return iter(self._sections.keys()) - - def __contains__(self, k): - return k in self._settings - - def __getitem__(self, k): - self._finalize() - - return self._sections[k] - - # Allow attribute access because it looks nice. - def __getattr__(self, k): - return self.__getitem__(k) diff --git a/python/mach/mach/decorators.py b/python/mach/mach/decorators.py deleted file mode 100644 index 733fd42f08c..00000000000 --- a/python/mach/mach/decorators.py +++ /dev/null @@ -1,349 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -from __future__ import absolute_import, unicode_literals - -import argparse -import collections -import inspect -import types - -from .base import MachError -from .config import ConfigProvider -from .registrar import Registrar - - -class _MachCommand(object): - """Container for mach command metadata. - - Mach commands contain lots of attributes. This class exists to capture them - in a sane way so tuples, etc aren't used instead. - """ - __slots__ = ( - # Content from decorator arguments to define the command. - 'name', - 'subcommand', - 'category', - 'description', - 'conditions', - '_parser', - 'arguments', - 'argument_group_names', - - # Describes how dispatch is performed. - - # The Python class providing the command. This is the class type not - # an instance of the class. Mach will instantiate a new instance of - # the class if the command is executed. - 'cls', - - # Whether the __init__ method of the class should receive a mach - # context instance. This should only affect the mach driver and how - # it instantiates classes. - 'pass_context', - - # The name of the method providing the command. In other words, this - # is the str name of the attribute on the class type corresponding to - # the name of the function. - 'method', - - # Dict of string to _MachCommand defining sub-commands for this - # command. - 'subcommand_handlers', - ) - - def __init__(self, name=None, subcommand=None, category=None, - description=None, conditions=None, parser=None): - self.name = name - self.subcommand = subcommand - self.category = category - self.description = description - self.conditions = conditions or [] - self._parser = parser - self.arguments = [] - self.argument_group_names = [] - - self.cls = None - self.pass_context = None - self.method = None - self.subcommand_handlers = {} - - @property - def parser(self): - # Creating CLI parsers at command dispatch time can be expensive. Make - # it possible to lazy load them by using functions. - if callable(self._parser): - self._parser = self._parser() - - return self._parser - - @property - def docstring(self): - return self.cls.__dict__[self.method].__doc__ - - def __ior__(self, other): - if not isinstance(other, _MachCommand): - raise ValueError('can only operate on _MachCommand instances') - - for a in self.__slots__: - if not getattr(self, a): - setattr(self, a, getattr(other, a)) - - return self - - -def CommandProvider(cls): - """Class decorator to denote that it provides subcommands for Mach. - - When this decorator is present, mach looks for commands being defined by - methods inside the class. - """ - - # The implementation of this decorator relies on the parse-time behavior of - # decorators. When the module is imported, the method decorators (like - # @Command and @CommandArgument) are called *before* this class decorator. - # The side-effect of the method decorators is to store specifically-named - # attributes on the function types. We just scan over all functions in the - # class looking for the side-effects of the method decorators. - - # Tell mach driver whether to pass context argument to __init__. - pass_context = False - - if inspect.ismethod(cls.__init__): - spec = inspect.getargspec(cls.__init__) - - if len(spec.args) > 2: - msg = 'Mach @CommandProvider class %s implemented incorrectly. ' + \ - '__init__() must take 1 or 2 arguments. From %s' - msg = msg % (cls.__name__, inspect.getsourcefile(cls)) - raise MachError(msg) - - if len(spec.args) == 2: - pass_context = True - - seen_commands = set() - - # We scan __dict__ because we only care about the classes own attributes, - # not inherited ones. If we did inherited attributes, we could potentially - # define commands multiple times. We also sort keys so commands defined in - # the same class are grouped in a sane order. - for attr in sorted(cls.__dict__.keys()): - value = cls.__dict__[attr] - - if not isinstance(value, types.FunctionType): - continue - - command = getattr(value, '_mach_command', None) - if not command: - continue - - # Ignore subcommands for now: we handle them later. - if command.subcommand: - continue - - seen_commands.add(command.name) - - if not command.conditions and Registrar.require_conditions: - continue - - msg = 'Mach command \'%s\' implemented incorrectly. ' + \ - 'Conditions argument must take a list ' + \ - 'of functions. Found %s instead.' - - if not isinstance(command.conditions, collections.Iterable): - msg = msg % (command.name, type(command.conditions)) - raise MachError(msg) - - for c in command.conditions: - if not hasattr(c, '__call__'): - msg = msg % (command.name, type(c)) - raise MachError(msg) - - command.cls = cls - command.method = attr - command.pass_context = pass_context - - Registrar.register_command_handler(command) - - # Now do another pass to get sub-commands. We do this in two passes so - # we can check the parent command existence without having to hold - # state and reconcile after traversal. - for attr in sorted(cls.__dict__.keys()): - value = cls.__dict__[attr] - - if not isinstance(value, types.FunctionType): - continue - - command = getattr(value, '_mach_command', None) - if not command: - continue - - # It is a regular command. - if not command.subcommand: - continue - - if command.name not in seen_commands: - raise MachError('Command referenced by sub-command does not ' - 'exist: %s' % command.name) - - if command.name not in Registrar.command_handlers: - continue - - command.cls = cls - command.method = attr - command.pass_context = pass_context - parent = Registrar.command_handlers[command.name] - - if parent._parser: - raise MachError('cannot declare sub commands against a command ' - 'that has a parser installed: %s' % command) - if command.subcommand in parent.subcommand_handlers: - raise MachError('sub-command already defined: %s' % command.subcommand) - - parent.subcommand_handlers[command.subcommand] = command - - return cls - - -class Command(object): - """Decorator for functions or methods that provide a mach command. - - The decorator accepts arguments that define basic attributes of the - command. The following arguments are recognized: - - category -- The string category to which this command belongs. Mach's - help will group commands by category. - - description -- A brief description of what the command does. - - parser -- an optional argparse.ArgumentParser instance or callable - that returns an argparse.ArgumentParser instance to use as the - basis for the command arguments. - - For example: - - @Command('foo', category='misc', description='Run the foo action') - def foo(self): - pass - """ - def __init__(self, name, **kwargs): - self._mach_command = _MachCommand(name=name, **kwargs) - - def __call__(self, func): - if not hasattr(func, '_mach_command'): - func._mach_command = _MachCommand() - - func._mach_command |= self._mach_command - - return func - -class SubCommand(object): - """Decorator for functions or methods that provide a sub-command. - - Mach commands can have sub-commands. e.g. ``mach command foo`` or - ``mach command bar``. Each sub-command has its own parser and is - effectively its own mach command. - - The decorator accepts arguments that define basic attributes of the - sub command: - - command -- The string of the command this sub command should be - attached to. - - subcommand -- The string name of the sub command to register. - - description -- A textual description for this sub command. - """ - def __init__(self, command, subcommand, description=None): - self._mach_command = _MachCommand(name=command, subcommand=subcommand, - description=description) - - def __call__(self, func): - if not hasattr(func, '_mach_command'): - func._mach_command = _MachCommand() - - func._mach_command |= self._mach_command - - return func - -class CommandArgument(object): - """Decorator for additional arguments to mach subcommands. - - This decorator should be used to add arguments to mach commands. Arguments - to the decorator are proxied to ArgumentParser.add_argument(). - - For example: - - @Command('foo', help='Run the foo action') - @CommandArgument('-b', '--bar', action='store_true', default=False, - help='Enable bar mode.') - def foo(self): - pass - """ - def __init__(self, *args, **kwargs): - if kwargs.get('nargs') == argparse.REMAINDER: - # These are the assertions we make in dispatcher.py about - # those types of CommandArguments. - assert len(args) == 1 - assert all(k in ('default', 'nargs', 'help', 'group') for k in kwargs) - self._command_args = (args, kwargs) - - def __call__(self, func): - if not hasattr(func, '_mach_command'): - func._mach_command = _MachCommand() - - func._mach_command.arguments.insert(0, self._command_args) - - return func - - -class CommandArgumentGroup(object): - """Decorator for additional argument groups to mach commands. - - This decorator should be used to add arguments groups to mach commands. - Arguments to the decorator are proxied to - ArgumentParser.add_argument_group(). - - For example: - - @Command('foo', helps='Run the foo action') - @CommandArgumentGroup('group1') - @CommandArgument('-b', '--bar', group='group1', action='store_true', - default=False, help='Enable bar mode.') - def foo(self): - pass - - The name should be chosen so that it makes sense as part of the phrase - 'Command Arguments for <name>' because that's how it will be shown in the - help message. - """ - def __init__(self, group_name): - self._group_name = group_name - - def __call__(self, func): - if not hasattr(func, '_mach_command'): - func._mach_command = _MachCommand() - - func._mach_command.argument_group_names.insert(0, self._group_name) - - return func - - -def SettingsProvider(cls): - """Class decorator to denote that this class provides Mach settings. - - When this decorator is encountered, the underlying class will automatically - be registered with the Mach registrar and will (likely) be hooked up to the - mach driver. - - This decorator is only allowed on mach.config.ConfigProvider classes. - """ - if not issubclass(cls, ConfigProvider): - raise MachError('@SettingsProvider encountered on class that does ' + - 'not derived from mach.config.ConfigProvider.') - - Registrar.register_settings_provider(cls) - - return cls - diff --git a/python/mach/mach/dispatcher.py b/python/mach/mach/dispatcher.py deleted file mode 100644 index 666c4a3f691..00000000000 --- a/python/mach/mach/dispatcher.py +++ /dev/null @@ -1,446 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -from __future__ import absolute_import, unicode_literals - -import argparse -import difflib -import sys - -from operator import itemgetter - -from .base import ( - MachError, - NoCommandError, - UnknownCommandError, - UnrecognizedArgumentError, -) - - -class CommandFormatter(argparse.HelpFormatter): - """Custom formatter to format just a subcommand.""" - - def add_usage(self, *args): - pass - - -class CommandAction(argparse.Action): - """An argparse action that handles mach commands. - - This class is essentially a reimplementation of argparse's sub-parsers - feature. We first tried to use sub-parsers. However, they were missing - features like grouping of commands (http://bugs.python.org/issue14037). - - The way this works involves light magic and a partial understanding of how - argparse works. - - Arguments registered with an argparse.ArgumentParser have an action - associated with them. An action is essentially a class that when called - does something with the encountered argument(s). This class is one of those - action classes. - - An instance of this class is created doing something like: - - parser.add_argument('command', action=CommandAction, registrar=r) - - Note that a mach.registrar.Registrar instance is passed in. The Registrar - holds information on all the mach commands that have been registered. - - When this argument is registered with the ArgumentParser, an instance of - this class is instantiated. One of the subtle but important things it does - is tell the argument parser that it's interested in *all* of the remaining - program arguments. So, when the ArgumentParser calls this action, we will - receive the command name plus all of its arguments. - - For more, read the docs in __call__. - """ - def __init__(self, option_strings, dest, required=True, default=None, - registrar=None, context=None): - # A proper API would have **kwargs here. However, since we are a little - # hacky, we intentionally omit it as a way of detecting potentially - # breaking changes with argparse's implementation. - # - # In a similar vein, default is passed in but is not needed, so we drop - # it. - argparse.Action.__init__(self, option_strings, dest, required=required, - help=argparse.SUPPRESS, nargs=argparse.REMAINDER) - - self._mach_registrar = registrar - self._context = context - - def __call__(self, parser, namespace, values, option_string=None): - """This is called when the ArgumentParser has reached our arguments. - - Since we always register ourselves with nargs=argparse.REMAINDER, - values should be a list of remaining arguments to parse. The first - argument should be the name of the command to invoke and all remaining - arguments are arguments for that command. - - The gist of the flow is that we look at the command being invoked. If - it's *help*, we handle that specially (because argparse's default help - handler isn't satisfactory). Else, we create a new, independent - ArgumentParser instance for just the invoked command (based on the - information contained in the command registrar) and feed the arguments - into that parser. We then merge the results with the main - ArgumentParser. - """ - if namespace.help: - # -h or --help is in the global arguments. - self._handle_main_help(parser, namespace.verbose) - sys.exit(0) - elif values: - command = values[0].lower() - args = values[1:] - if command == 'help': - if args and args[0] not in ['-h', '--help']: - # Make sure args[0] is indeed a command. - self._handle_command_help(parser, args[0]) - else: - self._handle_main_help(parser, namespace.verbose) - sys.exit(0) - elif '-h' in args or '--help' in args: - # -h or --help is in the command arguments. - if '--' in args: - # -- is in command arguments - if '-h' in args[:args.index('--')] or '--help' in args[:args.index('--')]: - # Honor -h or --help only if it appears before -- - self._handle_command_help(parser, command) - sys.exit(0) - else: - self._handle_command_help(parser, command) - sys.exit(0) - - - else: - raise NoCommandError() - - # Command suggestion - if command not in self._mach_registrar.command_handlers: - # Make sure we don't suggest any deprecated commands. - names = [h.name for h in self._mach_registrar.command_handlers.values() - if h.cls.__name__ == 'DeprecatedCommands'] - # We first try to look for a valid command that is very similar to the given command. - suggested_commands = difflib.get_close_matches(command, names, cutoff=0.8) - # If we find more than one matching command, or no command at all, we give command suggestions instead - # (with a lower matching threshold). All commands that start with the given command (for instance: 'mochitest-plain', - # 'mochitest-chrome', etc. for 'mochitest-') are also included. - if len(suggested_commands) != 1: - suggested_commands = set(difflib.get_close_matches(command, names, cutoff=0.5)) - suggested_commands |= {cmd for cmd in names if cmd.startswith(command)} - raise UnknownCommandError(command, 'run', suggested_commands) - sys.stderr.write("We're assuming the '%s' command is '%s' and we're executing it for you.\n\n" % (command, suggested_commands[0])) - command = suggested_commands[0] - - handler = self._mach_registrar.command_handlers.get(command) - - usage = '%(prog)s [global arguments] ' + command + \ - ' [command arguments]' - - subcommand = None - - # If there are sub-commands, parse the intent out immediately. - if handler.subcommand_handlers: - if not args: - self._handle_subcommand_main_help(parser, handler) - sys.exit(0) - elif len(args) == 1 and args[0] in ('help', '--help'): - self._handle_subcommand_main_help(parser, handler) - sys.exit(0) - # mach <command> help <subcommand> - elif len(args) == 2 and args[0] == 'help': - subcommand = args[1] - subhandler = handler.subcommand_handlers[subcommand] - self._handle_subcommand_help(parser, command, subcommand, subhandler) - sys.exit(0) - # We are running a sub command. - else: - subcommand = args[0] - if subcommand[0] == '-': - raise MachError('%s invoked improperly. A sub-command name ' - 'must be the first argument after the command name.' % - command) - - if subcommand not in handler.subcommand_handlers: - raise UnknownCommandError(subcommand, 'run', - handler.subcommand_handlers.keys()) - - handler = handler.subcommand_handlers[subcommand] - usage = '%(prog)s [global arguments] ' + command + ' ' + \ - subcommand + ' [command arguments]' - args.pop(0) - - # We create a new parser, populate it with the command's arguments, - # then feed all remaining arguments to it, merging the results - # with ourselves. This is essentially what argparse subparsers - # do. - - parser_args = { - 'add_help': False, - 'usage': usage, - } - - remainder = None - - if handler.parser: - subparser = handler.parser - subparser.context = self._context - for arg in subparser._actions[:]: - if arg.nargs == argparse.REMAINDER: - subparser._actions.remove(arg) - remainder = (arg.dest,), {'default': arg.default, - 'nargs': arg.nargs, - 'help': arg.help} - else: - subparser = argparse.ArgumentParser(**parser_args) - - for arg in handler.arguments: - # Remove our group keyword; it's not needed here. - group_name = arg[1].get('group') - if group_name: - del arg[1]['group'] - - if arg[1].get('nargs') == argparse.REMAINDER: - # parse_known_args expects all argparse.REMAINDER ('...') - # arguments to be all stuck together. Instead, we want them to - # pick any extra argument, wherever they are. - # Assume a limited CommandArgument for those arguments. - assert len(arg[0]) == 1 - assert all(k in ('default', 'nargs', 'help') for k in arg[1]) - remainder = arg - else: - subparser.add_argument(*arg[0], **arg[1]) - - # We define the command information on the main parser result so as to - # not interfere with arguments passed to the command. - setattr(namespace, 'mach_handler', handler) - setattr(namespace, 'command', command) - setattr(namespace, 'subcommand', subcommand) - - command_namespace, extra = subparser.parse_known_args(args) - setattr(namespace, 'command_args', command_namespace) - if remainder: - (name,), options = remainder - # parse_known_args usefully puts all arguments after '--' in - # extra, but also puts '--' there. We don't want to pass it down - # to the command handler. Note that if multiple '--' are on the - # command line, only the first one is removed, so that subsequent - # ones are passed down. - if '--' in extra: - extra.remove('--') - - # Commands with argparse.REMAINDER arguments used to force the - # other arguments to be '+' prefixed. If a user now passes such - # an argument, if will silently end up in extra. So, check if any - # of the allowed arguments appear in a '+' prefixed form, and error - # out if that's the case. - for args, _ in handler.arguments: - for arg in args: - arg = arg.replace('-', '+', 1) - if arg in extra: - raise UnrecognizedArgumentError(command, [arg]) - - if extra: - setattr(command_namespace, name, extra) - else: - setattr(command_namespace, name, options.get('default', [])) - elif extra and handler.cls.__name__ != 'DeprecatedCommands': - raise UnrecognizedArgumentError(command, extra) - - def _handle_main_help(self, parser, verbose): - # Since we don't need full sub-parser support for the main help output, - # we create groups in the ArgumentParser and populate each group with - # arguments corresponding to command names. This has the side-effect - # that argparse renders it nicely. - r = self._mach_registrar - disabled_commands = [] - - cats = [(k, v[2]) for k, v in r.categories.items()] - sorted_cats = sorted(cats, key=itemgetter(1), reverse=True) - for category, priority in sorted_cats: - group = None - - for command in sorted(r.commands_by_category[category]): - handler = r.command_handlers[command] - - # Instantiate a handler class to see if it should be filtered - # out for the current context or not. Condition functions can be - # applied to the command's decorator. - if handler.conditions: - if handler.pass_context: - instance = handler.cls(self._context) - else: - instance = handler.cls() - - is_filtered = False - for c in handler.conditions: - if not c(instance): - is_filtered = True - break - if is_filtered: - description = handler.description - disabled_command = {'command': command, 'description': description} - disabled_commands.append(disabled_command) - continue - - if group is None: - title, description, _priority = r.categories[category] - group = parser.add_argument_group(title, description) - - description = handler.description - group.add_argument(command, help=description, - action='store_true') - - if disabled_commands and 'disabled' in r.categories: - title, description, _priority = r.categories['disabled'] - group = parser.add_argument_group(title, description) - if verbose == True: - for c in disabled_commands: - group.add_argument(c['command'], help=c['description'], - action='store_true') - - parser.print_help() - - def _populate_command_group(self, parser, handler, group): - extra_groups = {} - for group_name in handler.argument_group_names: - group_full_name = 'Command Arguments for ' + group_name - extra_groups[group_name] = \ - parser.add_argument_group(group_full_name) - - for arg in handler.arguments: - # Apply our group keyword. - group_name = arg[1].get('group') - if group_name: - del arg[1]['group'] - group = extra_groups[group_name] - group.add_argument(*arg[0], **arg[1]) - - def _handle_command_help(self, parser, command): - handler = self._mach_registrar.command_handlers.get(command) - - if not handler: - raise UnknownCommandError(command, 'query') - - if handler.subcommand_handlers: - self._handle_subcommand_main_help(parser, handler) - return - - # This code is worth explaining. Because we are doing funky things with - # argument registration to allow the same option in both global and - # command arguments, we can't simply put all arguments on the same - # parser instance because argparse would complain. We can't register an - # argparse subparser here because it won't properly show help for - # global arguments. So, we employ a strategy similar to command - # execution where we construct a 2nd, independent ArgumentParser for - # just the command data then supplement the main help's output with - # this 2nd parser's. We use a custom formatter class to ignore some of - # the help output. - parser_args = { - 'formatter_class': CommandFormatter, - 'add_help': False, - } - - if handler.parser: - c_parser = handler.parser - c_parser.context = self._context - c_parser.formatter_class = NoUsageFormatter - # Accessing _action_groups is a bit shady. We are highly dependent - # on the argparse implementation not changing. We fail fast to - # detect upstream changes so we can intelligently react to them. - group = c_parser._action_groups[1] - - # By default argparse adds two groups called "positional arguments" - # and "optional arguments". We want to rename these to reflect standard - # mach terminology. - c_parser._action_groups[0].title = 'Command Parameters' - c_parser._action_groups[1].title = 'Command Arguments' - - if not handler.description: - handler.description = c_parser.description - c_parser.description = None - else: - c_parser = argparse.ArgumentParser(**parser_args) - group = c_parser.add_argument_group('Command Arguments') - - self._populate_command_group(c_parser, handler, group) - - # Set the long help of the command to the docstring (if present) or - # the command decorator description argument (if present). - if handler.docstring: - parser.description = format_docstring(handler.docstring) - elif handler.description: - parser.description = handler.description - - parser.usage = '%(prog)s [global arguments] ' + command + \ - ' [command arguments]' - - # This is needed to preserve line endings in the description field, - # which may be populated from a docstring. - parser.formatter_class = argparse.RawDescriptionHelpFormatter - parser.print_help() - print('') - c_parser.print_help() - - def _handle_subcommand_main_help(self, parser, handler): - parser.usage = '%(prog)s [global arguments] ' + handler.name + \ - ' subcommand [subcommand arguments]' - group = parser.add_argument_group('Sub Commands') - - for subcommand, subhandler in sorted(handler.subcommand_handlers.iteritems()): - group.add_argument(subcommand, help=subhandler.description, - action='store_true') - - if handler.docstring: - parser.description = format_docstring(handler.docstring) - - parser.formatter_class = argparse.RawDescriptionHelpFormatter - - parser.print_help() - - def _handle_subcommand_help(self, parser, command, subcommand, handler): - parser.usage = '%(prog)s [global arguments] ' + command + \ - ' ' + subcommand + ' [command arguments]' - - c_parser = argparse.ArgumentParser(add_help=False, - formatter_class=CommandFormatter) - group = c_parser.add_argument_group('Sub Command Arguments') - self._populate_command_group(c_parser, handler, group) - - if handler.docstring: - parser.description = format_docstring(handler.docstring) - - parser.formatter_class = argparse.RawDescriptionHelpFormatter - - parser.print_help() - print('') - c_parser.print_help() - - -class NoUsageFormatter(argparse.HelpFormatter): - def _format_usage(self, *args, **kwargs): - return "" - - -def format_docstring(docstring): - """Format a raw docstring into something suitable for presentation. - - This function is based on the example function in PEP-0257. - """ - if not docstring: - return '' - lines = docstring.expandtabs().splitlines() - indent = sys.maxint - for line in lines[1:]: - stripped = line.lstrip() - if stripped: - indent = min(indent, len(line) - len(stripped)) - trimmed = [lines[0].strip()] - if indent < sys.maxint: - for line in lines[1:]: - trimmed.append(line[indent:].rstrip()) - while trimmed and not trimmed[-1]: - trimmed.pop() - while trimmed and not trimmed[0]: - trimmed.pop(0) - return '\n'.join(trimmed) diff --git a/python/mach/mach/logging.py b/python/mach/mach/logging.py deleted file mode 100644 index 729e6cb3d93..00000000000 --- a/python/mach/mach/logging.py +++ /dev/null @@ -1,256 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at http://mozilla.org/MPL/2.0/. - -# This file contains logging functionality for mach. It essentially provides -# support for a structured logging framework built on top of Python's built-in -# logging framework. - -from __future__ import absolute_import, unicode_literals - -try: - import blessings -except ImportError: - blessings = None - -import json -import logging -import sys -import time - - -def format_seconds(total): - """Format number of seconds to MM:SS.DD form.""" - - minutes, seconds = divmod(total, 60) - - return '%2d:%05.2f' % (minutes, seconds) - - -class ConvertToStructuredFilter(logging.Filter): - """Filter that converts unstructured records into structured ones.""" - def filter(self, record): - if hasattr(record, 'action') and hasattr(record, 'params'): - return True - - record.action = 'unstructured' - record.params = {'msg': record.getMessage()} - record.msg = '{msg}' - - return True - - -class StructuredJSONFormatter(logging.Formatter): - """Log formatter that writes a structured JSON entry.""" - - def format(self, record): - action = getattr(record, 'action', 'UNKNOWN') - params = getattr(record, 'params', {}) - - return json.dumps([record.created, action, params]) - - -class StructuredHumanFormatter(logging.Formatter): - """Log formatter that writes structured messages for humans. - - It is important that this formatter never be added to a logger that - produces unstructured/classic log messages. If it is, the call to format() - could fail because the string could contain things (like JSON) that look - like formatting character sequences. - - Because of this limitation, format() will fail with a KeyError if an - unstructured record is passed or if the structured message is malformed. - """ - def __init__(self, start_time, write_interval=False, write_times=True): - self.start_time = start_time - self.write_interval = write_interval - self.write_times = write_times - self.last_time = None - - def format(self, record): - f = record.msg.format(**record.params) - - if not self.write_times: - return f - - elapsed = self._time(record) - - return '%s %s' % (format_seconds(elapsed), f) - - def _time(self, record): - t = record.created - self.start_time - - if self.write_interval and self.last_time is not None: - t = record.created - self.last_time - - self.last_time = record.created - - return t - - -class StructuredTerminalFormatter(StructuredHumanFormatter): - """Log formatter for structured messages writing to a terminal.""" - - def set_terminal(self, terminal): - self.terminal = terminal - - def format(self, record): - f = record.msg.format(**record.params) - - if not self.write_times: - return f - - t = self.terminal.blue(format_seconds(self._time(record))) - - return '%s %s' % (t, self._colorize(f)) - - def _colorize(self, s): - if not self.terminal: - return s - - result = s - - reftest = s.startswith('REFTEST ') - if reftest: - s = s[8:] - - if s.startswith('TEST-PASS'): - result = self.terminal.green(s[0:9]) + s[9:] - elif s.startswith('TEST-UNEXPECTED'): - result = self.terminal.red(s[0:20]) + s[20:] - elif s.startswith('TEST-START'): - result = self.terminal.yellow(s[0:10]) + s[10:] - elif s.startswith('TEST-INFO'): - result = self.terminal.yellow(s[0:9]) + s[9:] - - if reftest: - result = 'REFTEST ' + result - - return result - - -class LoggingManager(object): - """Holds and controls global logging state. - - An application should instantiate one of these and configure it as needed. - - This class provides a mechanism to configure the output of logging data - both from mach and from the overall logging system (e.g. from other - modules). - """ - - def __init__(self): - self.start_time = time.time() - - self.json_handlers = [] - self.terminal_handler = None - self.terminal_formatter = None - - self.root_logger = logging.getLogger() - self.root_logger.setLevel(logging.DEBUG) - - # Installing NullHandler on the root logger ensures that *all* log - # messages have at least one handler. This prevents Python from - # complaining about "no handlers could be found for logger XXX." - self.root_logger.addHandler(logging.NullHandler()) - - self.mach_logger = logging.getLogger('mach') - self.mach_logger.setLevel(logging.DEBUG) - - self.structured_filter = ConvertToStructuredFilter() - - self.structured_loggers = [self.mach_logger] - - self._terminal = None - - @property - def terminal(self): - if not self._terminal and blessings: - # Sometimes blessings fails to set up the terminal. In that case, - # silently fail. - try: - terminal = blessings.Terminal(stream=sys.stdout) - - if terminal.is_a_tty: - self._terminal = terminal - except Exception: - pass - - return self._terminal - - def add_json_handler(self, fh): - """Enable JSON logging on the specified file object.""" - - # Configure the consumer of structured messages. - handler = logging.StreamHandler(stream=fh) - handler.setFormatter(StructuredJSONFormatter()) - handler.setLevel(logging.DEBUG) - - # And hook it up. - for logger in self.structured_loggers: - logger.addHandler(handler) - - self.json_handlers.append(handler) - - def add_terminal_logging(self, fh=sys.stdout, level=logging.INFO, - write_interval=False, write_times=True): - """Enable logging to the terminal.""" - - formatter = StructuredHumanFormatter(self.start_time, - write_interval=write_interval, write_times=write_times) - - if self.terminal: - formatter = StructuredTerminalFormatter(self.start_time, - write_interval=write_interval, write_times=write_times) - formatter.set_terminal(self.terminal) - - handler = logging.StreamHandler(stream=fh) - handler.setFormatter(formatter) - handler.setLevel(level) - - for logger in self.structured_loggers: - logger.addHandler(handler) - - self.terminal_handler = handler - self.terminal_formatter = formatter - - def replace_terminal_handler(self, handler): - """Replace the installed terminal handler. - - Returns the old handler or None if none was configured. - If the new handler is None, removes any existing handler and disables - logging to the terminal. - """ - old = self.terminal_handler - - if old: - for logger in self.structured_loggers: - logger.removeHandler(old) - - if handler: - for logger in self.structured_loggers: - logger.addHandler(handler) - - self.terminal_handler = handler - - return old - - def enable_unstructured(self): - """Enable logging of unstructured messages.""" - if self.terminal_handler: - self.terminal_handler.addFilter(self.structured_filter) - self.root_logger.addHandler(self.terminal_handler) - - def disable_unstructured(self): - """Disable logging of unstructured messages.""" - if self.terminal_handler: - self.terminal_handler.removeFilter(self.structured_filter) - self.root_logger.removeHandler(self.terminal_handler) - - def register_structured_logger(self, logger): - """Register a structured logger. - - This needs to be called for all structured loggers that don't chain up - to the mach logger in order for their output to be captured. - """ - self.structured_loggers.append(logger) diff --git a/python/mach/mach/main.py b/python/mach/mach/main.py deleted file mode 100644 index 479c619200f..00000000000 --- a/python/mach/mach/main.py +++ /dev/null @@ -1,575 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -# This module provides functionality for the command-line build tool -# (mach). It is packaged as a module because everything is a library. - -from __future__ import absolute_import, print_function, unicode_literals -from collections import Iterable - -import argparse -import codecs -import imp -import logging -import os -import sys -import traceback -import uuid - -from .base import ( - CommandContext, - MachError, - NoCommandError, - UnknownCommandError, - UnrecognizedArgumentError, -) - -from .decorators import ( - CommandArgument, - CommandProvider, - Command, -) - -from .config import ConfigSettings -from .dispatcher import CommandAction -from .logging import LoggingManager -from .registrar import Registrar - - - -MACH_ERROR = r''' -The error occurred in mach itself. This is likely a bug in mach itself or a -fundamental problem with a loaded module. - -Please consider filing a bug against mach by going to the URL: - - https://bugzilla.mozilla.org/enter_bug.cgi?product=Core&component=mach - -'''.lstrip() - -ERROR_FOOTER = r''' -If filing a bug, please include the full output of mach, including this error -message. - -The details of the failure are as follows: -'''.lstrip() - -COMMAND_ERROR = r''' -The error occurred in the implementation of the invoked mach command. - -This should never occur and is likely a bug in the implementation of that -command. Consider filing a bug for this issue. -'''.lstrip() - -MODULE_ERROR = r''' -The error occurred in code that was called by the mach command. This is either -a bug in the called code itself or in the way that mach is calling it. - -You should consider filing a bug for this issue. -'''.lstrip() - -NO_COMMAND_ERROR = r''' -It looks like you tried to run mach without a command. - -Run |mach help| to show a list of commands. -'''.lstrip() - -UNKNOWN_COMMAND_ERROR = r''' -It looks like you are trying to %s an unknown mach command: %s -%s -Run |mach help| to show a list of commands. -'''.lstrip() - -SUGGESTED_COMMANDS_MESSAGE = r''' -Did you want to %s any of these commands instead: %s? -''' - -UNRECOGNIZED_ARGUMENT_ERROR = r''' -It looks like you passed an unrecognized argument into mach. - -The %s command does not accept the arguments: %s -'''.lstrip() - -INVALID_ENTRY_POINT = r''' -Entry points should return a list of command providers or directories -containing command providers. The following entry point is invalid: - - %s - -You are seeing this because there is an error in an external module attempting -to implement a mach command. Please fix the error, or uninstall the module from -your system. -'''.lstrip() - -class ArgumentParser(argparse.ArgumentParser): - """Custom implementation argument parser to make things look pretty.""" - - def error(self, message): - """Custom error reporter to give more helpful text on bad commands.""" - if not message.startswith('argument command: invalid choice'): - argparse.ArgumentParser.error(self, message) - assert False - - print('Invalid command specified. The list of commands is below.\n') - self.print_help() - sys.exit(1) - - def format_help(self): - text = argparse.ArgumentParser.format_help(self) - - # Strip out the silly command list that would preceed the pretty list. - # - # Commands: - # {foo,bar} - # foo Do foo. - # bar Do bar. - search = 'Commands:\n {' - start = text.find(search) - - if start != -1: - end = text.find('}\n', start) - assert end != -1 - - real_start = start + len('Commands:\n') - real_end = end + len('}\n') - - text = text[0:real_start] + text[real_end:] - - return text - - -class ContextWrapper(object): - def __init__(self, context, handler): - object.__setattr__(self, '_context', context) - object.__setattr__(self, '_handler', handler) - - def __getattribute__(self, key): - try: - return getattr(object.__getattribute__(self, '_context'), key) - except AttributeError as e: - try: - ret = object.__getattribute__(self, '_handler')(self, key) - except (AttributeError, TypeError): - # TypeError is in case the handler comes from old code not - # taking a key argument. - raise e - setattr(self, key, ret) - return ret - - def __setattr__(self, key, value): - setattr(object.__getattribute__(self, '_context'), key, value) - - -@CommandProvider -class Mach(object): - """Main mach driver type. - - This type is responsible for holding global mach state and dispatching - a command from arguments. - - The following attributes may be assigned to the instance to influence - behavior: - - populate_context_handler -- If defined, it must be a callable. The - callable signature is the following: - populate_context_handler(context, key=None) - It acts as a fallback getter for the mach.base.CommandContext - instance. - This allows to augment the context instance with arbitrary data - for use in command handlers. - For backwards compatibility, it is also called before command - dispatch without a key, allowing the context handler to add - attributes to the context instance. - - require_conditions -- If True, commands that do not have any condition - functions applied will be skipped. Defaults to False. - - """ - - USAGE = """%(prog)s [global arguments] command [command arguments] - -mach (German for "do") is the main interface to the Mozilla build system and -common developer tasks. - -You tell mach the command you want to perform and it does it for you. - -Some common commands are: - - %(prog)s build Build/compile the source tree. - %(prog)s help Show full help, including the list of all commands. - -To see more help for a specific command, run: - - %(prog)s help <command> -""" - - def __init__(self, cwd): - assert os.path.isdir(cwd) - - self.cwd = cwd - self.log_manager = LoggingManager() - self.logger = logging.getLogger(__name__) - self.settings = ConfigSettings() - - self.log_manager.register_structured_logger(self.logger) - self.global_arguments = [] - self.populate_context_handler = None - - def add_global_argument(self, *args, **kwargs): - """Register a global argument with the argument parser. - - Arguments are proxied to ArgumentParser.add_argument() - """ - - self.global_arguments.append((args, kwargs)) - - def load_commands_from_directory(self, path): - """Scan for mach commands from modules in a directory. - - This takes a path to a directory, loads the .py files in it, and - registers and found mach command providers with this mach instance. - """ - for f in sorted(os.listdir(path)): - if not f.endswith('.py') or f == '__init__.py': - continue - - full_path = os.path.join(path, f) - module_name = 'mach.commands.%s' % f[0:-3] - - self.load_commands_from_file(full_path, module_name=module_name) - - def load_commands_from_file(self, path, module_name=None): - """Scan for mach commands from a file. - - This takes a path to a file and loads it as a Python module under the - module name specified. If no name is specified, a random one will be - chosen. - """ - if module_name is None: - # Ensure parent module is present otherwise we'll (likely) get - # an error due to unknown parent. - if b'mach.commands' not in sys.modules: - mod = imp.new_module(b'mach.commands') - sys.modules[b'mach.commands'] = mod - - module_name = 'mach.commands.%s' % uuid.uuid1().get_hex() - - imp.load_source(module_name, path) - - def load_commands_from_entry_point(self, group='mach.providers'): - """Scan installed packages for mach command provider entry points. An - entry point is a function that returns a list of paths to files or - directories containing command providers. - - This takes an optional group argument which specifies the entry point - group to use. If not specified, it defaults to 'mach.providers'. - """ - try: - import pkg_resources - except ImportError: - print("Could not find setuptools, ignoring command entry points", - file=sys.stderr) - return - - for entry in pkg_resources.iter_entry_points(group=group, name=None): - paths = entry.load()() - if not isinstance(paths, Iterable): - print(INVALID_ENTRY_POINT % entry) - sys.exit(1) - - for path in paths: - if os.path.isfile(path): - self.load_commands_from_file(path) - elif os.path.isdir(path): - self.load_commands_from_directory(path) - else: - print("command provider '%s' does not exist" % path) - - def define_category(self, name, title, description, priority=50): - """Provide a description for a named command category.""" - - Registrar.register_category(name, title, description, priority) - - @property - def require_conditions(self): - return Registrar.require_conditions - - @require_conditions.setter - def require_conditions(self, value): - Registrar.require_conditions = value - - def run(self, argv, stdin=None, stdout=None, stderr=None): - """Runs mach with arguments provided from the command line. - - Returns the integer exit code that should be used. 0 means success. All - other values indicate failure. - """ - - # If no encoding is defined, we default to UTF-8 because without this - # Python 2.7 will assume the default encoding of ASCII. This will blow - # up with UnicodeEncodeError as soon as it encounters a non-ASCII - # character in a unicode instance. We simply install a wrapper around - # the streams and restore once we have finished. - stdin = sys.stdin if stdin is None else stdin - stdout = sys.stdout if stdout is None else stdout - stderr = sys.stderr if stderr is None else stderr - - orig_stdin = sys.stdin - orig_stdout = sys.stdout - orig_stderr = sys.stderr - - sys.stdin = stdin - sys.stdout = stdout - sys.stderr = stderr - - try: - if stdin.encoding is None: - sys.stdin = codecs.getreader('utf-8')(stdin) - - if stdout.encoding is None: - sys.stdout = codecs.getwriter('utf-8')(stdout) - - if stderr.encoding is None: - sys.stderr = codecs.getwriter('utf-8')(stderr) - - return self._run(argv) - except KeyboardInterrupt: - print('mach interrupted by signal or user action. Stopping.') - return 1 - - except Exception as e: - # _run swallows exceptions in invoked handlers and converts them to - # a proper exit code. So, the only scenario where we should get an - # exception here is if _run itself raises. If _run raises, that's a - # bug in mach (or a loaded command module being silly) and thus - # should be reported differently. - self._print_error_header(argv, sys.stdout) - print(MACH_ERROR) - - exc_type, exc_value, exc_tb = sys.exc_info() - stack = traceback.extract_tb(exc_tb) - - self._print_exception(sys.stdout, exc_type, exc_value, stack) - - return 1 - - finally: - sys.stdin = orig_stdin - sys.stdout = orig_stdout - sys.stderr = orig_stderr - - def _run(self, argv): - context = CommandContext(cwd=self.cwd, - settings=self.settings, log_manager=self.log_manager, - commands=Registrar) - - if self.populate_context_handler: - self.populate_context_handler(context) - context = ContextWrapper(context, self.populate_context_handler) - - parser = self.get_argument_parser(context) - - if not len(argv): - # We don't register the usage until here because if it is globally - # registered, argparse always prints it. This is not desired when - # running with --help. - parser.usage = Mach.USAGE - parser.print_usage() - return 0 - - try: - args = parser.parse_args(argv) - except NoCommandError: - print(NO_COMMAND_ERROR) - return 1 - except UnknownCommandError as e: - suggestion_message = SUGGESTED_COMMANDS_MESSAGE % (e.verb, ', '.join(e.suggested_commands)) if e.suggested_commands else '' - print(UNKNOWN_COMMAND_ERROR % (e.verb, e.command, suggestion_message)) - return 1 - except UnrecognizedArgumentError as e: - print(UNRECOGNIZED_ARGUMENT_ERROR % (e.command, - ' '.join(e.arguments))) - return 1 - - # Add JSON logging to a file if requested. - if args.logfile: - self.log_manager.add_json_handler(args.logfile) - - # Up the logging level if requested. - log_level = logging.INFO - if args.verbose: - log_level = logging.DEBUG - - self.log_manager.register_structured_logger(logging.getLogger('mach')) - - write_times = True - if args.log_no_times or 'MACH_NO_WRITE_TIMES' in os.environ: - write_times = False - - # Always enable terminal logging. The log manager figures out if we are - # actually in a TTY or are a pipe and does the right thing. - self.log_manager.add_terminal_logging(level=log_level, - write_interval=args.log_interval, write_times=write_times) - - self.load_settings(args) - - if not hasattr(args, 'mach_handler'): - raise MachError('ArgumentParser result missing mach handler info.') - - handler = getattr(args, 'mach_handler') - - try: - return Registrar._run_command_handler(handler, context=context, - debug_command=args.debug_command, **vars(args.command_args)) - except KeyboardInterrupt as ki: - raise ki - except Exception as e: - exc_type, exc_value, exc_tb = sys.exc_info() - - # The first two frames are us and are never used. - stack = traceback.extract_tb(exc_tb)[2:] - - # If we have nothing on the stack, the exception was raised as part - # of calling the @Command method itself. This likely means a - # mismatch between @CommandArgument and arguments to the method. - # e.g. there exists a @CommandArgument without the corresponding - # argument on the method. We handle that here until the module - # loader grows the ability to validate better. - if not len(stack): - print(COMMAND_ERROR) - self._print_exception(sys.stdout, exc_type, exc_value, - traceback.extract_tb(exc_tb)) - return 1 - - # Split the frames into those from the module containing the - # command and everything else. - command_frames = [] - other_frames = [] - - initial_file = stack[0][0] - - for frame in stack: - if frame[0] == initial_file: - command_frames.append(frame) - else: - other_frames.append(frame) - - # If the exception was in the module providing the command, it's - # likely the bug is in the mach command module, not something else. - # If there are other frames, the bug is likely not the mach - # command's fault. - self._print_error_header(argv, sys.stdout) - - if len(other_frames): - print(MODULE_ERROR) - else: - print(COMMAND_ERROR) - - self._print_exception(sys.stdout, exc_type, exc_value, stack) - - return 1 - - def log(self, level, action, params, format_str): - """Helper method to record a structured log event.""" - self.logger.log(level, format_str, - extra={'action': action, 'params': params}) - - def _print_error_header(self, argv, fh): - fh.write('Error running mach:\n\n') - fh.write(' ') - fh.write(repr(argv)) - fh.write('\n\n') - - def _print_exception(self, fh, exc_type, exc_value, stack): - fh.write(ERROR_FOOTER) - fh.write('\n') - - for l in traceback.format_exception_only(exc_type, exc_value): - fh.write(l) - - fh.write('\n') - for l in traceback.format_list(stack): - fh.write(l) - - def load_settings(self, args): - """Determine which settings files apply and load them. - - Currently, we only support loading settings from a single file. - Ideally, we support loading from multiple files. This is supported by - the ConfigSettings API. However, that API currently doesn't track where - individual values come from, so if we load from multiple sources then - save, we effectively do a full copy. We don't want this. Until - ConfigSettings does the right thing, we shouldn't expose multi-file - loading. - - We look for a settings file in the following locations. The first one - found wins: - - 1) Command line argument - 2) Environment variable - 3) Default path - """ - # Settings are disabled until integration with command providers is - # worked out. - self.settings = None - return False - - for provider in Registrar.settings_providers: - provider.register_settings() - self.settings.register_provider(provider) - - p = os.path.join(self.cwd, 'mach.ini') - - if args.settings_file: - p = args.settings_file - elif 'MACH_SETTINGS_FILE' in os.environ: - p = os.environ['MACH_SETTINGS_FILE'] - - self.settings.load_file(p) - - return os.path.exists(p) - - def get_argument_parser(self, context): - """Returns an argument parser for the command-line interface.""" - - parser = ArgumentParser(add_help=False, - usage='%(prog)s [global arguments] command [command arguments]') - - # Order is important here as it dictates the order the auto-generated - # help messages are printed. - global_group = parser.add_argument_group('Global Arguments') - - #global_group.add_argument('--settings', dest='settings_file', - # metavar='FILENAME', help='Path to settings file.') - - global_group.add_argument('-v', '--verbose', dest='verbose', - action='store_true', default=False, - help='Print verbose output.') - global_group.add_argument('-l', '--log-file', dest='logfile', - metavar='FILENAME', type=argparse.FileType('ab'), - help='Filename to write log data to.') - global_group.add_argument('--log-interval', dest='log_interval', - action='store_true', default=False, - help='Prefix log line with interval from last message rather ' - 'than relative time. Note that this is NOT execution time ' - 'if there are parallel operations.') - global_group.add_argument('--log-no-times', dest='log_no_times', - action='store_true', default=False, - help='Do not prefix log lines with times. By default, mach will ' - 'prefix each output line with the time since command start.') - global_group.add_argument('-h', '--help', dest='help', - action='store_true', default=False, - help='Show this help message.') - global_group.add_argument('--debug-command', action='store_true', - help='Start a Python debugger when command is dispatched.') - - for args, kwargs in self.global_arguments: - global_group.add_argument(*args, **kwargs) - - # We need to be last because CommandAction swallows all remaining - # arguments and argparse parses arguments in the order they were added. - parser.add_argument('command', action=CommandAction, - registrar=Registrar, context=context) - - return parser diff --git a/python/mach/mach/mixin/__init__.py b/python/mach/mach/mixin/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/python/mach/mach/mixin/__init__.py +++ /dev/null diff --git a/python/mach/mach/mixin/logging.py b/python/mach/mach/mixin/logging.py deleted file mode 100644 index 5c37b54f1bd..00000000000 --- a/python/mach/mach/mixin/logging.py +++ /dev/null @@ -1,55 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -from __future__ import absolute_import, unicode_literals - -import logging - - -class LoggingMixin(object): - """Provides functionality to control logging.""" - - def populate_logger(self, name=None): - """Ensure this class instance has a logger associated with it. - - Users of this mixin that call log() will need to ensure self._logger is - a logging.Logger instance before they call log(). This function ensures - self._logger is defined by populating it if it isn't. - """ - if hasattr(self, '_logger'): - return - - if name is None: - name = '.'.join([self.__module__, self.__class__.__name__]) - - self._logger = logging.getLogger(name) - - def log(self, level, action, params, format_str): - """Log a structured log event. - - A structured log event consists of a logging level, a string action, a - dictionary of attributes, and a formatting string. - - The logging level is one of the logging.* constants, such as - logging.INFO. - - The action string is essentially the enumeration of the event. Each - different type of logged event should have a different action. - - The params dict is the metadata constituting the logged event. - - The formatting string is used to convert the structured message back to - human-readable format. Conversion back to human-readable form is - performed by calling format() on this string, feeding into it the dict - of attributes constituting the event. - - Example Usage - ------------- - - self.log(logging.DEBUG, 'login', {'username': 'johndoe'}, - 'User login: {username}') - """ - self._logger.log(level, format_str, - extra={'action': action, 'params': params}) - diff --git a/python/mach/mach/mixin/process.py b/python/mach/mach/mixin/process.py deleted file mode 100644 index 5b06da3b60e..00000000000 --- a/python/mach/mach/mixin/process.py +++ /dev/null @@ -1,175 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -# This module provides mixins to perform process execution. - -from __future__ import absolute_import, unicode_literals - -import logging -import os -import subprocess -import sys - -from mozprocess.processhandler import ProcessHandlerMixin - -from .logging import LoggingMixin - - -# Perform detection of operating system environment. This is used by command -# execution. We only do this once to save redundancy. Yes, this can fail module -# loading. That is arguably OK. -if 'SHELL' in os.environ: - _current_shell = os.environ['SHELL'] -elif 'MOZILLABUILD' in os.environ: - _current_shell = os.environ['MOZILLABUILD'] + '/msys/bin/sh.exe' -elif 'COMSPEC' in os.environ: - _current_shell = os.environ['COMSPEC'] -else: - raise Exception('Could not detect environment shell!') - -_in_msys = False - -if os.environ.get('MSYSTEM', None) == 'MINGW32': - _in_msys = True - - if not _current_shell.lower().endswith('.exe'): - _current_shell += '.exe' - - -class ProcessExecutionMixin(LoggingMixin): - """Mix-in that provides process execution functionality.""" - - def run_process(self, args=None, cwd=None, append_env=None, - explicit_env=None, log_name=None, log_level=logging.INFO, - line_handler=None, require_unix_environment=False, - ensure_exit_code=0, ignore_children=False, pass_thru=False): - """Runs a single process to completion. - - Takes a list of arguments to run where the first item is the - executable. Runs the command in the specified directory and - with optional environment variables. - - append_env -- Dict of environment variables to append to the current - set of environment variables. - explicit_env -- Dict of environment variables to set for the new - process. Any existing environment variables will be ignored. - - require_unix_environment if True will ensure the command is executed - within a UNIX environment. Basically, if we are on Windows, it will - execute the command via an appropriate UNIX-like shell. - - ignore_children is proxied to mozprocess's ignore_children. - - ensure_exit_code is used to ensure the exit code of a process matches - what is expected. If it is an integer, we raise an Exception if the - exit code does not match this value. If it is True, we ensure the exit - code is 0. If it is False, we don't perform any exit code validation. - - pass_thru is a special execution mode where the child process inherits - this process's standard file handles (stdin, stdout, stderr) as well as - additional file descriptors. It should be used for interactive processes - where buffering from mozprocess could be an issue. pass_thru does not - use mozprocess. Therefore, arguments like log_name, line_handler, - and ignore_children have no effect. - """ - args = self._normalize_command(args, require_unix_environment) - - self.log(logging.INFO, 'new_process', {'args': args}, ' '.join(args)) - - def handleLine(line): - # Converts str to unicode on Python 2 and bytes to str on Python 3. - if isinstance(line, bytes): - line = line.decode(sys.stdout.encoding or 'utf-8', 'replace') - - if line_handler: - line_handler(line) - - if not log_name: - return - - self.log(log_level, log_name, {'line': line.rstrip()}, '{line}') - - use_env = {} - if explicit_env: - use_env = explicit_env - else: - use_env.update(os.environ) - - if append_env: - use_env.update(append_env) - - self.log(logging.DEBUG, 'process', {'env': use_env}, 'Environment: {env}') - - # There is a bug in subprocess where it doesn't like unicode types in - # environment variables. Here, ensure all unicode are converted to - # binary. utf-8 is our globally assumed default. If the caller doesn't - # want UTF-8, they shouldn't pass in a unicode instance. - normalized_env = {} - for k, v in use_env.items(): - if isinstance(k, unicode): - k = k.encode('utf-8', 'strict') - - if isinstance(v, unicode): - v = v.encode('utf-8', 'strict') - - normalized_env[k] = v - - use_env = normalized_env - - if pass_thru: - proc = subprocess.Popen(args, cwd=cwd, env=use_env) - status = None - # Leave it to the subprocess to handle Ctrl+C. If it terminates as - # a result of Ctrl+C, proc.wait() will return a status code, and, - # we get out of the loop. If it doesn't, like e.g. gdb, we continue - # waiting. - while status is None: - try: - status = proc.wait() - except KeyboardInterrupt: - pass - else: - p = ProcessHandlerMixin(args, cwd=cwd, env=use_env, - processOutputLine=[handleLine], universal_newlines=True, - ignore_children=ignore_children) - p.run() - p.processOutput() - status = p.wait() - - if ensure_exit_code is False: - return status - - if ensure_exit_code is True: - ensure_exit_code = 0 - - if status != ensure_exit_code: - raise Exception('Process executed with non-0 exit code: %s' % args) - - return status - - def _normalize_command(self, args, require_unix_environment): - """Adjust command arguments to run in the necessary environment. - - This exists mainly to facilitate execution of programs requiring a *NIX - shell when running on Windows. The caller specifies whether a shell - environment is required. If it is and we are running on Windows but - aren't running in the UNIX-like msys environment, then we rewrite the - command to execute via a shell. - """ - assert isinstance(args, list) and len(args) - - if not require_unix_environment or not _in_msys: - return args - - # Always munge Windows-style into Unix style for the command. - prog = args[0].replace('\\', '/') - - # PyMake removes the C: prefix. But, things seem to work here - # without it. Not sure what that's about. - - # We run everything through the msys shell. We need to use - # '-c' and pass all the arguments as one argument because that is - # how sh works. - cline = subprocess.list2cmdline([prog] + args[1:]) - return [_current_shell, '-c', cline] diff --git a/python/mach/mach/registrar.py b/python/mach/mach/registrar.py deleted file mode 100644 index 522f761dcee..00000000000 --- a/python/mach/mach/registrar.py +++ /dev/null @@ -1,119 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -from __future__ import absolute_import, unicode_literals - -from .base import MachError - -INVALID_COMMAND_CONTEXT = r''' -It looks like you tried to run a mach command from an invalid context. The %s -command failed to meet the following conditions: %s - -Run |mach help| to show a list of all commands available to the current context. -'''.lstrip() - - -class MachRegistrar(object): - """Container for mach command and config providers.""" - - def __init__(self): - self.command_handlers = {} - self.commands_by_category = {} - self.settings_providers = set() - self.categories = {} - self.require_conditions = False - - def register_command_handler(self, handler): - name = handler.name - - if not handler.category: - raise MachError('Cannot register a mach command without a ' - 'category: %s' % name) - - if handler.category not in self.categories: - raise MachError('Cannot register a command to an undefined ' - 'category: %s -> %s' % (name, handler.category)) - - self.command_handlers[name] = handler - self.commands_by_category[handler.category].add(name) - - def register_settings_provider(self, cls): - self.settings_providers.add(cls) - - def register_category(self, name, title, description, priority=50): - self.categories[name] = (title, description, priority) - self.commands_by_category[name] = set() - - @classmethod - def _condition_failed_message(cls, name, conditions): - msg = ['\n'] - for c in conditions: - part = [' %s' % c.__name__] - if c.__doc__ is not None: - part.append(c.__doc__) - msg.append(' - '.join(part)) - return INVALID_COMMAND_CONTEXT % (name, '\n'.join(msg)) - - def _run_command_handler(self, handler, context=None, debug_command=False, **kwargs): - cls = handler.cls - - if handler.pass_context and not context: - raise Exception('mach command class requires context.') - - if context: - prerun = getattr(context, 'pre_dispatch_handler', None) - if prerun: - prerun(context, handler, args=kwargs) - - if handler.pass_context: - instance = cls(context) - else: - instance = cls() - - if handler.conditions: - fail_conditions = [] - for c in handler.conditions: - if not c(instance): - fail_conditions.append(c) - - if fail_conditions: - print(self._condition_failed_message(handler.name, fail_conditions)) - return 1 - - fn = getattr(instance, handler.method) - - if debug_command: - import pdb - result = pdb.runcall(fn, **kwargs) - else: - result = fn(**kwargs) - - result = result or 0 - assert isinstance(result, (int, long)) - return result - - def dispatch(self, name, context=None, argv=None, **kwargs): - """Dispatch/run a command. - - Commands can use this to call other commands. - """ - # TODO handler.subcommand_handlers are ignored - handler = self.command_handlers[name] - - if handler.parser: - parser = handler.parser - - # save and restore existing defaults so **kwargs don't persist across - # subsequent invocations of Registrar.dispatch() - old_defaults = parser._defaults.copy() - parser.set_defaults(**kwargs) - kwargs, _ = parser.parse_known_args(argv or []) - kwargs = vars(kwargs) - parser._defaults = old_defaults - - return self._run_command_handler(handler, context=context, **kwargs) - - - -Registrar = MachRegistrar() diff --git a/python/mach/mach/terminal.py b/python/mach/mach/terminal.py deleted file mode 100644 index 9115211e021..00000000000 --- a/python/mach/mach/terminal.py +++ /dev/null @@ -1,75 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -"""This file contains code for interacting with terminals. - -All the terminal interaction code is consolidated so the complexity can be in -one place, away from code that is commonly looked at. -""" - -from __future__ import absolute_import, print_function, unicode_literals - -import logging -import sys - - -class LoggingHandler(logging.Handler): - """Custom logging handler that works with terminal window dressing. - - This is alternative terminal logging handler which contains smarts for - emitting terminal control characters properly. Currently, it has generic - support for "footer" elements at the bottom of the screen. Functionality - can be added when needed. - """ - def __init__(self): - logging.Handler.__init__(self) - - self.fh = sys.stdout - self.footer = None - - def flush(self): - self.acquire() - - try: - self.fh.flush() - finally: - self.release() - - def emit(self, record): - msg = self.format(record) - - if self.footer: - self.footer.clear() - - self.fh.write(msg) - self.fh.write('\n') - - if self.footer: - self.footer.draw() - - # If we don't flush, the footer may not get drawn. - self.flush() - - -class TerminalFooter(object): - """Represents something drawn on the bottom of a terminal.""" - def __init__(self, terminal): - self.t = terminal - self.fh = sys.stdout - - def _clear_lines(self, n): - for i in xrange(n): - self.fh.write(self.t.move_x(0)) - self.fh.write(self.t.clear_eol()) - self.fh.write(self.t.move_up()) - - self.fh.write(self.t.move_down()) - self.fh.write(self.t.move_x(0)) - - def clear(self): - raise Exception('clear() must be implemented.') - - def draw(self): - raise Exception('draw() must be implemented.') - diff --git a/python/mach/mach/test/__init__.py b/python/mach/mach/test/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/python/mach/mach/test/__init__.py +++ /dev/null diff --git a/python/mach/mach/test/common.py b/python/mach/mach/test/common.py deleted file mode 100644 index 1c4b1ea90ac..00000000000 --- a/python/mach/mach/test/common.py +++ /dev/null @@ -1,40 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -from __future__ import unicode_literals - -from StringIO import StringIO -import os -import unittest - -from mach.main import Mach -from mach.base import CommandContext - -here = os.path.abspath(os.path.dirname(__file__)) - -class TestBase(unittest.TestCase): - provider_dir = os.path.join(here, 'providers') - - def _run_mach(self, args, provider_file=None, entry_point=None, context_handler=None): - m = Mach(os.getcwd()) - m.define_category('testing', 'Mach unittest', 'Testing for mach core', 10) - m.populate_context_handler = context_handler - - if provider_file: - m.load_commands_from_file(os.path.join(self.provider_dir, provider_file)) - - if entry_point: - m.load_commands_from_entry_point(entry_point) - - stdout = StringIO() - stderr = StringIO() - stdout.encoding = 'UTF-8' - stderr.encoding = 'UTF-8' - - try: - result = m.run(args, stdout=stdout, stderr=stderr) - except SystemExit: - result = None - - return (result, stdout.getvalue(), stderr.getvalue()) diff --git a/python/mach/mach/test/providers/__init__.py b/python/mach/mach/test/providers/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/python/mach/mach/test/providers/__init__.py +++ /dev/null diff --git a/python/mach/mach/test/providers/basic.py b/python/mach/mach/test/providers/basic.py deleted file mode 100644 index d10856289b1..00000000000 --- a/python/mach/mach/test/providers/basic.py +++ /dev/null @@ -1,15 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import unicode_literals - -from mach.decorators import ( - CommandProvider, - Command, -) - -@CommandProvider -class ConditionsProvider(object): - @Command('cmd_foo', category='testing') - def run_foo(self): - pass diff --git a/python/mach/mach/test/providers/conditions.py b/python/mach/mach/test/providers/conditions.py deleted file mode 100644 index a95429752d4..00000000000 --- a/python/mach/mach/test/providers/conditions.py +++ /dev/null @@ -1,53 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -from __future__ import unicode_literals - -from mach.decorators import ( - CommandProvider, - Command, -) - -def is_foo(cls): - """Foo must be true""" - return cls.foo - -def is_bar(cls): - """Bar must be true""" - return cls.bar - -@CommandProvider -class ConditionsProvider(object): - foo = True - bar = False - - @Command('cmd_foo', category='testing', conditions=[is_foo]) - def run_foo(self): - pass - - @Command('cmd_bar', category='testing', conditions=[is_bar]) - def run_bar(self): - pass - - @Command('cmd_foobar', category='testing', conditions=[is_foo, is_bar]) - def run_foobar(self): - pass - -@CommandProvider -class ConditionsContextProvider(object): - def __init__(self, context): - self.foo = context.foo - self.bar = context.bar - - @Command('cmd_foo_ctx', category='testing', conditions=[is_foo]) - def run_foo(self): - pass - - @Command('cmd_bar_ctx', category='testing', conditions=[is_bar]) - def run_bar(self): - pass - - @Command('cmd_foobar_ctx', category='testing', conditions=[is_foo, is_bar]) - def run_foobar(self): - pass diff --git a/python/mach/mach/test/providers/conditions_invalid.py b/python/mach/mach/test/providers/conditions_invalid.py deleted file mode 100644 index 22284d4dcad..00000000000 --- a/python/mach/mach/test/providers/conditions_invalid.py +++ /dev/null @@ -1,16 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -from __future__ import unicode_literals - -from mach.decorators import ( - CommandProvider, - Command, -) - -@CommandProvider -class ConditionsProvider(object): - @Command('cmd_foo', category='testing', conditions=["invalid"]) - def run_foo(self): - pass diff --git a/python/mach/mach/test/providers/throw.py b/python/mach/mach/test/providers/throw.py deleted file mode 100644 index 06bee01eec7..00000000000 --- a/python/mach/mach/test/providers/throw.py +++ /dev/null @@ -1,29 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -from __future__ import unicode_literals - -import time - -from mach.decorators import ( - CommandArgument, - CommandProvider, - Command, -) - -from mach.test.providers import throw2 - - -@CommandProvider -class TestCommandProvider(object): - @Command('throw', category='testing') - @CommandArgument('--message', '-m', default='General Error') - def throw(self, message): - raise Exception(message) - - @Command('throw_deep', category='testing') - @CommandArgument('--message', '-m', default='General Error') - def throw_deep(self, message): - throw2.throw_deep(message) - diff --git a/python/mach/mach/test/providers/throw2.py b/python/mach/mach/test/providers/throw2.py deleted file mode 100644 index af0a23fcfe7..00000000000 --- a/python/mach/mach/test/providers/throw2.py +++ /dev/null @@ -1,13 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -# This file exists to trigger the differences in mach error reporting between -# exceptions that occur in mach command modules themselves and in the things -# they call. - -def throw_deep(message): - return throw_real(message) - -def throw_real(message): - raise Exception(message) diff --git a/python/mach/mach/test/test_conditions.py b/python/mach/mach/test/test_conditions.py deleted file mode 100644 index 20080687e39..00000000000 --- a/python/mach/mach/test/test_conditions.py +++ /dev/null @@ -1,83 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -from __future__ import unicode_literals - -import os - -from mach.base import MachError -from mach.main import Mach -from mach.registrar import Registrar -from mach.test.common import TestBase - -from mozunit import main - - -def _populate_context(context, key=None): - if key is None: - return - if key == 'foo': - return True - if key == 'bar': - return False - raise AttributeError(key) - -class TestConditions(TestBase): - """Tests for conditionally filtering commands.""" - - def _run_mach(self, args, context_handler=None): - return TestBase._run_mach(self, args, 'conditions.py', - context_handler=context_handler) - - - def test_conditions_pass(self): - """Test that a command which passes its conditions is runnable.""" - - self.assertEquals((0, '', ''), self._run_mach(['cmd_foo'])) - self.assertEquals((0, '', ''), self._run_mach(['cmd_foo_ctx'], _populate_context)) - - def test_invalid_context_message(self): - """Test that commands which do not pass all their conditions - print the proper failure message.""" - - def is_bar(): - """Bar must be true""" - fail_conditions = [is_bar] - - for name in ('cmd_bar', 'cmd_foobar'): - result, stdout, stderr = self._run_mach([name]) - self.assertEquals(1, result) - - fail_msg = Registrar._condition_failed_message(name, fail_conditions) - self.assertEquals(fail_msg.rstrip(), stdout.rstrip()) - - for name in ('cmd_bar_ctx', 'cmd_foobar_ctx'): - result, stdout, stderr = self._run_mach([name], _populate_context) - self.assertEquals(1, result) - - fail_msg = Registrar._condition_failed_message(name, fail_conditions) - self.assertEquals(fail_msg.rstrip(), stdout.rstrip()) - - def test_invalid_type(self): - """Test that a condition which is not callable raises an exception.""" - - m = Mach(os.getcwd()) - m.define_category('testing', 'Mach unittest', 'Testing for mach core', 10) - self.assertRaises(MachError, m.load_commands_from_file, - os.path.join(self.provider_dir, 'conditions_invalid.py')) - - def test_help_message(self): - """Test that commands that are not runnable do not show up in help.""" - - result, stdout, stderr = self._run_mach(['help'], _populate_context) - self.assertIn('cmd_foo', stdout) - self.assertNotIn('cmd_bar', stdout) - self.assertNotIn('cmd_foobar', stdout) - self.assertIn('cmd_foo_ctx', stdout) - self.assertNotIn('cmd_bar_ctx', stdout) - self.assertNotIn('cmd_foobar_ctx', stdout) - - -if __name__ == '__main__': - main() diff --git a/python/mach/mach/test/test_config.py b/python/mach/mach/test/test_config.py deleted file mode 100644 index cebd47a7d6c..00000000000 --- a/python/mach/mach/test/test_config.py +++ /dev/null @@ -1,264 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import unicode_literals - -import sys -import unittest - -from mozfile.mozfile import NamedTemporaryFile - -from mach.config import ( - AbsolutePathType, - BooleanType, - ConfigProvider, - ConfigSettings, - IntegerType, - PathType, - PositiveIntegerType, - RelativePathType, - StringType, -) - -from mozunit import main - - -if sys.version_info[0] == 3: - str_type = str -else: - str_type = basestring - -CONFIG1 = r""" -[foo] - -bar = bar_value -baz = /baz/foo.c -""" - -CONFIG2 = r""" -[foo] - -bar = value2 -""" - -class Provider1(ConfigProvider): - @classmethod - def _register_settings(cls): - cls.register_setting('foo', 'bar', StringType) - cls.register_setting('foo', 'baz', AbsolutePathType) - -Provider1.register_settings() - -class ProviderDuplicate(ConfigProvider): - @classmethod - def _register_settings(cls): - cls.register_setting('dupesect', 'foo', StringType) - cls.register_setting('dupesect', 'foo', StringType) - -class TestConfigProvider(unittest.TestCase): - def test_construct(self): - s = Provider1.config_settings - - self.assertEqual(len(s), 1) - self.assertIn('foo', s) - - self.assertEqual(len(s['foo']), 2) - self.assertIn('bar', s['foo']) - self.assertIn('baz', s['foo']) - - def test_duplicate_option(self): - with self.assertRaises(Exception): - ProviderDuplicate.register_settings() - - -class Provider2(ConfigProvider): - @classmethod - def _register_settings(cls): - cls.register_setting('a', 'string', StringType) - cls.register_setting('a', 'boolean', BooleanType) - cls.register_setting('a', 'pos_int', PositiveIntegerType) - cls.register_setting('a', 'int', IntegerType) - cls.register_setting('a', 'abs_path', AbsolutePathType) - cls.register_setting('a', 'rel_path', RelativePathType) - cls.register_setting('a', 'path', PathType) - -Provider2.register_settings() - -class TestConfigSettings(unittest.TestCase): - def test_empty(self): - s = ConfigSettings() - - self.assertEqual(len(s), 0) - self.assertNotIn('foo', s) - - def test_simple(self): - s = ConfigSettings() - s.register_provider(Provider1) - - self.assertEqual(len(s), 1) - self.assertIn('foo', s) - - foo = s['foo'] - foo = s.foo - - self.assertEqual(len(foo), 2) - - self.assertIn('bar', foo) - self.assertIn('baz', foo) - - foo['bar'] = 'value1' - self.assertEqual(foo['bar'], 'value1') - self.assertEqual(foo['bar'], 'value1') - - def test_assignment_validation(self): - s = ConfigSettings() - s.register_provider(Provider2) - - a = s.a - - # Assigning an undeclared setting raises. - with self.assertRaises(KeyError): - a.undefined = True - - with self.assertRaises(KeyError): - a['undefined'] = True - - # Basic type validation. - a.string = 'foo' - a.string = 'foo' - - with self.assertRaises(TypeError): - a.string = False - - a.boolean = True - a.boolean = False - - with self.assertRaises(TypeError): - a.boolean = 'foo' - - a.pos_int = 5 - a.pos_int = 0 - - with self.assertRaises(ValueError): - a.pos_int = -1 - - with self.assertRaises(TypeError): - a.pos_int = 'foo' - - a.int = 5 - a.int = 0 - a.int = -5 - - with self.assertRaises(TypeError): - a.int = 1.24 - - with self.assertRaises(TypeError): - a.int = 'foo' - - a.abs_path = '/home/gps' - - with self.assertRaises(ValueError): - a.abs_path = 'home/gps' - - a.rel_path = 'home/gps' - a.rel_path = './foo/bar' - a.rel_path = 'foo.c' - - with self.assertRaises(ValueError): - a.rel_path = '/foo/bar' - - a.path = '/home/gps' - a.path = 'foo.c' - a.path = 'foo/bar' - a.path = './foo' - - def test_retrieval_type(self): - s = ConfigSettings() - s.register_provider(Provider2) - - a = s.a - - a.string = 'foo' - a.boolean = True - a.pos_int = 12 - a.int = -4 - a.abs_path = '/home/gps' - a.rel_path = 'foo.c' - a.path = './foo/bar' - - self.assertIsInstance(a.string, str_type) - self.assertIsInstance(a.boolean, bool) - self.assertIsInstance(a.pos_int, int) - self.assertIsInstance(a.int, int) - self.assertIsInstance(a.abs_path, str_type) - self.assertIsInstance(a.rel_path, str_type) - self.assertIsInstance(a.path, str_type) - - def test_file_reading_single(self): - temp = NamedTemporaryFile(mode='wt') - temp.write(CONFIG1) - temp.flush() - - s = ConfigSettings() - s.register_provider(Provider1) - - s.load_file(temp.name) - - self.assertEqual(s.foo.bar, 'bar_value') - - def test_file_reading_multiple(self): - """Loading multiple files has proper overwrite behavior.""" - temp1 = NamedTemporaryFile(mode='wt') - temp1.write(CONFIG1) - temp1.flush() - - temp2 = NamedTemporaryFile(mode='wt') - temp2.write(CONFIG2) - temp2.flush() - - s = ConfigSettings() - s.register_provider(Provider1) - - s.load_files([temp1.name, temp2.name]) - - self.assertEqual(s.foo.bar, 'value2') - - def test_file_reading_missing(self): - """Missing files should silently be ignored.""" - - s = ConfigSettings() - - s.load_file('/tmp/foo.ini') - - def test_file_writing(self): - s = ConfigSettings() - s.register_provider(Provider2) - - s.a.string = 'foo' - s.a.boolean = False - - temp = NamedTemporaryFile('wt') - s.write(temp) - temp.flush() - - s2 = ConfigSettings() - s2.register_provider(Provider2) - - s2.load_file(temp.name) - - self.assertEqual(s.a.string, s2.a.string) - self.assertEqual(s.a.boolean, s2.a.boolean) - - def test_write_pot(self): - s = ConfigSettings() - s.register_provider(Provider1) - s.register_provider(Provider2) - - # Just a basic sanity test. - temp = NamedTemporaryFile('wt') - s.write_pot(temp) - temp.flush() - - -if __name__ == '__main__': - main() diff --git a/python/mach/mach/test/test_entry_point.py b/python/mach/mach/test/test_entry_point.py deleted file mode 100644 index 5bd2c279d45..00000000000 --- a/python/mach/mach/test/test_entry_point.py +++ /dev/null @@ -1,60 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import unicode_literals - -import imp -import os -import sys - -from mach.base import MachError -from mach.test.common import TestBase -from mock import patch - -from mozunit import main - - -here = os.path.abspath(os.path.dirname(__file__)) - -class Entry(): - """Stub replacement for pkg_resources.EntryPoint""" - def __init__(self, providers): - self.providers = providers - - def load(self): - def _providers(): - return self.providers - return _providers - -class TestEntryPoints(TestBase): - """Test integrating with setuptools entry points""" - provider_dir = os.path.join(here, 'providers') - - def _run_mach(self): - return TestBase._run_mach(self, ['help'], entry_point='mach.providers') - - @patch('pkg_resources.iter_entry_points') - def test_load_entry_point_from_directory(self, mock): - # Ensure parent module is present otherwise we'll (likely) get - # an error due to unknown parent. - if b'mach.commands' not in sys.modules: - mod = imp.new_module(b'mach.commands') - sys.modules[b'mach.commands'] = mod - - mock.return_value = [Entry(['providers'])] - # Mach error raised due to conditions_invalid.py - with self.assertRaises(MachError): - self._run_mach() - - @patch('pkg_resources.iter_entry_points') - def test_load_entry_point_from_file(self, mock): - mock.return_value = [Entry([os.path.join('providers', 'basic.py')])] - - result, stdout, stderr = self._run_mach() - self.assertIsNone(result) - self.assertIn('cmd_foo', stdout) - - -# Not enabled in automation because tests are failing. -#if __name__ == '__main__': -# main() diff --git a/python/mach/mach/test/test_error_output.py b/python/mach/mach/test/test_error_output.py deleted file mode 100644 index 25553f96bc4..00000000000 --- a/python/mach/mach/test/test_error_output.py +++ /dev/null @@ -1,39 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -from __future__ import unicode_literals - -from mach.main import ( - COMMAND_ERROR, - MODULE_ERROR -) -from mach.test.common import TestBase - -from mozunit import main - - -class TestErrorOutput(TestBase): - - def _run_mach(self, args): - return TestBase._run_mach(self, args, 'throw.py') - - def test_command_error(self): - result, stdout, stderr = self._run_mach(['throw', '--message', - 'Command Error']) - - self.assertEqual(result, 1) - - self.assertIn(COMMAND_ERROR, stdout) - - def test_invoked_error(self): - result, stdout, stderr = self._run_mach(['throw_deep', '--message', - 'Deep stack']) - - self.assertEqual(result, 1) - - self.assertIn(MODULE_ERROR, stdout) - - -if __name__ == '__main__': - main() diff --git a/python/mach/mach/test/test_logger.py b/python/mach/mach/test/test_logger.py deleted file mode 100644 index 05592845e7f..00000000000 --- a/python/mach/mach/test/test_logger.py +++ /dev/null @@ -1,47 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -from __future__ import absolute_import, unicode_literals - -import logging -import time -import unittest - -from mach.logging import StructuredHumanFormatter - -from mozunit import main - - -class DummyLogger(logging.Logger): - def __init__(self, cb): - logging.Logger.__init__(self, 'test') - - self._cb = cb - - def handle(self, record): - self._cb(record) - - -class TestStructuredHumanFormatter(unittest.TestCase): - def test_non_ascii_logging(self): - # Ensures the formatter doesn't choke when non-ASCII characters are - # present in printed parameters. - formatter = StructuredHumanFormatter(time.time()) - - def on_record(record): - result = formatter.format(record) - relevant = result[9:] - - self.assertEqual(relevant, 'Test: s\xe9curit\xe9') - - logger = DummyLogger(on_record) - - value = 's\xe9curit\xe9' - - logger.log(logging.INFO, 'Test: {utf}', - extra={'action': 'action', 'params': {'utf': value}}) - - -if __name__ == '__main__': - main() diff --git a/python/mach/setup.py b/python/mach/setup.py deleted file mode 100644 index 511a6a32277..00000000000 --- a/python/mach/setup.py +++ /dev/null @@ -1,38 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -try: - from setuptools import setup -except: - from distutils.core import setup - - -VERSION = '0.3' - -README = open('README.rst').read() - -setup( - name='mach', - description='Generic command line command dispatching framework.', - long_description=README, - license='MPL 2.0', - author='Gregory Szorc', - author_email='gregory.szorc@gmail.com', - url='https://developer.mozilla.org/en-US/docs/Developer_Guide/mach', - packages=['mach'], - version=VERSION, - classifiers=[ - 'Environment :: Console', - 'Development Status :: 3 - Alpha', - 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)', - 'Natural Language :: English', - ], - install_requires=[ - 'blessings', - 'mozfile', - 'mozprocess', - ], - tests_require=['mock'], -) - diff --git a/python/mach_bootstrap.py b/python/mach_bootstrap.py index e77c78e0851..57e6472e5c8 100644 --- a/python/mach_bootstrap.py +++ b/python/mach_bootstrap.py @@ -12,7 +12,6 @@ from distutils.spawn import find_executable from pipes import quote SEARCH_PATHS = [ - os.path.join("python", "mach"), os.path.join("python", "tidy"), os.path.join("tests", "wpt"), os.path.join("tests", "wpt", "harness"), diff --git a/python/requirements.txt b/python/requirements.txt index 4a2d9bbd480..cf6ddc136b0 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,6 +1,5 @@ -# 'mach' is not listed here because a new version hasn't been published to PyPi in a while - blessings == 1.6 +mach == 0.6.0 mozdebug == 0.1 mozinfo == 0.8 mozlog == 3.0 |