diff options
author | Jack Moffitt <jack@metajack.im> | 2014-08-28 09:34:23 -0600 |
---|---|---|
committer | Jack Moffitt <jack@metajack.im> | 2014-09-08 20:21:42 -0600 |
commit | c6ab60dbfc6da7b4f800c9e40893c8b58413960c (patch) | |
tree | d1d74076cf7fa20e4f77ec7cb82cae98b67362cb /python/mach | |
parent | db2f642c32fc5bed445bb6f2e45b0f6f0b4342cf (diff) | |
download | servo-c6ab60dbfc6da7b4f800c9e40893c8b58413960c.tar.gz servo-c6ab60dbfc6da7b4f800c9e40893c8b58413960c.zip |
Cargoify servo
Diffstat (limited to 'python/mach')
31 files changed, 3436 insertions, 0 deletions
diff --git a/python/mach/README.rst b/python/mach/README.rst new file mode 100644 index 00000000000..25e8fd470bc --- /dev/null +++ b/python/mach/README.rst @@ -0,0 +1,328 @@ +==== +mach +==== + +Mach (German for *do*) is a generic command dispatcher for the command +line. + +To use mach, you install the mach core (a Python package), create an +executable *driver* script (named whatever you want), and write mach +commands. When the *driver* is executed, mach dispatches to the +requested command handler automatically. + +Features +======== + +On a high level, mach is similar to using argparse with subparsers (for +command handling). When you dig deeper, mach offers a number of +additional features: + +Distributed command definitions + With optparse/argparse, you have to define your commands on a central + parser instance. With mach, you annotate your command methods with + decorators and mach finds and dispatches to them automatically. + +Command categories + Mach commands can be grouped into categories when displayed in help. + This is currently not possible with argparse. + +Logging management + Mach provides a facility for logging (both classical text and + structured) that is available to any command handler. + +Settings files + Mach provides a facility for reading settings from an ini-like file + format. + +Components +========== + +Mach is conceptually composed of the following components: + +core + The mach core is the core code powering mach. This is a Python package + that contains all the business logic that makes mach work. The mach + core is common to all mach deployments. + +commands + These are what mach dispatches to. Commands are simply Python methods + registered as command names. The set of commands is unique to the + environment mach is deployed in. + +driver + The *driver* is the entry-point to mach. It is simply an executable + script that loads the mach core, tells it where commands can be found, + then asks the mach core to handle the current request. The driver is + unique to the deployed environment. But, it's usually based on an + example from this source tree. + +Project State +============= + +mach was originally written as a command dispatching framework to aid +Firefox development. While the code is mostly generic, there are still +some pieces that closely tie it to Mozilla/Firefox. The goal is for +these to eventually be removed and replaced with generic features so +mach is suitable for anybody to use. Until then, mach may not be the +best fit for you. + +Implementing Commands +--------------------- + +Mach commands are defined via Python decorators. + +All the relevant decorators are defined in the *mach.decorators* module. +The important decorators are as follows: + +CommandProvider + A class decorator that denotes that a class contains mach + commands. The decorator takes no arguments. + +Command + A method decorator that denotes that the method should be called when + the specified command is requested. The decorator takes a command name + as its first argument and a number of additional arguments to + configure the behavior of the command. + +CommandArgument + A method decorator that defines an argument to the command. Its + arguments are essentially proxied to ArgumentParser.add_argument() + +Classes with the *@CommandProvider* decorator *must* have an *__init__* +method that accepts 1 or 2 arguments. If it accepts 2 arguments, the +2nd argument will be a *MachCommandContext* instance. This is just a named +tuple containing references to objects provided by the mach driver. + +Here is a complete example:: + + from mach.decorators import ( + CommandArgument, + CommandProvider, + Command, + ) + + @CommandProvider + class MyClass(object): + @Command('doit', help='Do ALL OF THE THINGS.') + @CommandArgument('--force', '-f', action='store_true', + help='Force doing it.') + def doit(self, force=False): + # Do stuff here. + +When the module is loaded, the decorators tell mach about all handlers. +When mach runs, it takes the assembled metadata from these handlers and +hooks it up to the command line driver. Under the hood, arguments passed +to the decorators are being used to help mach parse command arguments, +formulate arguments to the methods, etc. See the documentation in the +*mach.base* module for more. + +The Python modules defining mach commands do not need to live inside the +main mach source tree. + +Conditionally Filtering Commands +-------------------------------- + +Sometimes it might only make sense to run a command given a certain +context. For example, running tests only makes sense if the product +they are testing has been built, and said build is available. To make +sure a command is only runnable from within a correct context, you can +define a series of conditions on the *Command* decorator. + +A condition is simply a function that takes an instance of the +*CommandProvider* class as an argument, and returns True or False. If +any of the conditions defined on a command return False, the command +will not be runnable. The doc string of a condition function is used in +error messages, to explain why the command cannot currently be run. + +Here is an example: + + from mach.decorators import ( + CommandProvider, + Command, + ) + + def build_available(cls): + """The build needs to be available.""" + return cls.build_path is not None + + @CommandProvider + class MyClass(MachCommandBase): + def __init__(self, build_path=None): + self.build_path = build_path + + @Command('run_tests', conditions=[build_available]) + def run_tests(self): + # Do stuff here. + +It is important to make sure that any state needed by the condition is +available to instances of the command provider. + +By default all commands without any conditions applied will be runnable, +but it is possible to change this behaviour by setting *require_conditions* +to True: + + m = mach.main.Mach() + m.require_conditions = True + +Minimizing Code in Commands +--------------------------- + +Mach command modules, classes, and methods work best when they are +minimal dispatchers. The reason is import bloat. Currently, the mach +core needs to import every Python file potentially containing mach +commands for every command invocation. If you have dozens of commands or +commands in modules that import a lot of Python code, these imports +could slow mach down and waste memory. + +It is thus recommended that mach modules, classes, and methods do as +little work as possible. Ideally the module should only import from +the *mach* package. If you need external modules, you should import them +from within the command method. + +To keep code size small, the body of a command method should be limited +to: + +1. Obtaining user input (parsing arguments, prompting, etc) +2. Calling into some other Python package +3. Formatting output + +Of course, these recommendations can be ignored if you want to risk +slower performance. + +In the future, the mach driver may cache the dispatching information or +have it intelligently loaded to facilitate lazy loading. + +Logging +======= + +Mach configures a built-in logging facility so commands can easily log +data. + +What sets the logging facility apart from most loggers you've seen is +that it encourages structured logging. Instead of conventional logging +where simple strings are logged, the internal logging mechanism logs all +events with the following pieces of information: + +* A string *action* +* A dict of log message fields +* A formatting string + +Essentially, instead of assembling a human-readable string at +logging-time, you create an object holding all the pieces of data that +will constitute your logged event. For each unique type of logged event, +you assign an *action* name. + +Depending on how logging is configured, your logged event could get +written a couple of different ways. + +JSON Logging +------------ + +Where machines are the intended target of the logging data, a JSON +logger is configured. The JSON logger assembles an array consisting of +the following elements: + +* Decimal wall clock time in seconds since UNIX epoch +* String *action* of message +* Object with structured message data + +The JSON-serialized array is written to a configured file handle. +Consumers of this logging stream can just perform a readline() then feed +that into a JSON deserializer to reconstruct the original logged +message. They can key off the *action* element to determine how to +process individual events. There is no need to invent a parser. +Convenient, isn't it? + +Logging for Humans +------------------ + +Where humans are the intended consumer of a log message, the structured +log message are converted to more human-friendly form. This is done by +utilizing the *formatting* string provided at log time. The logger +simply calls the *format* method of the formatting string, passing the +dict containing the message's fields. + +When *mach* is used in a terminal that supports it, the logging facility +also supports terminal features such as colorization. This is done +automatically in the logging layer - there is no need to control this at +logging time. + +In addition, messages intended for humans typically prepends every line +with the time passed since the application started. + +Logging HOWTO +------------- + +Structured logging piggybacks on top of Python's built-in logging +infrastructure provided by the *logging* package. We accomplish this by +taking advantage of *logging.Logger.log()*'s *extra* argument. To this +argument, we pass a dict with the fields *action* and *params*. These +are the string *action* and dict of message fields, respectively. The +formatting string is passed as the *msg* argument, like normal. + +If you were logging to a logger directly, you would do something like: + + logger.log(logging.INFO, 'My name is {name}', + extra={'action': 'my_name', 'params': {'name': 'Gregory'}}) + +The JSON logging would produce something like: + + [1339985554.306338, "my_name", {"name": "Gregory"}] + +Human logging would produce something like: + + 0.52 My name is Gregory + +Since there is a lot of complexity using logger.log directly, it is +recommended to go through a wrapping layer that hides part of the +complexity for you. The easiest way to do this is by utilizing the +LoggingMixin: + + import logging + from mach.mixin.logging import LoggingMixin + + class MyClass(LoggingMixin): + def foo(self): + self.log(logging.INFO, 'foo_start', {'bar': True}, + 'Foo performed. Bar: {bar}') + +Entry Points +============ + +It is possible to use setuptools' entry points to load commands +directly from python packages. A mach entry point is a function which +returns a list of files or directories containing mach command +providers. e.g.:: + + def list_providers(): + providers = [] + here = os.path.abspath(os.path.dirname(__file__)) + for p in os.listdir(here): + if p.endswith('.py'): + providers.append(os.path.join(here, p)) + return providers + +See http://pythonhosted.org/setuptools/setuptools.html#dynamic-discovery-of-services-and-plugins +for more information on creating an entry point. To search for entry +point plugins, you can call *load_commands_from_entry_point*. This +takes a single parameter called *group*. This is the name of the entry +point group to load and defaults to ``mach.providers``. e.g.:: + + mach.load_commands_from_entry_point("mach.external.providers") + +Adding Global Arguments +======================= + +Arguments to mach commands are usually command-specific. However, +mach ships with a handful of global arguments that apply to all +commands. + +It is possible to extend the list of global arguments. In your +*mach driver*, simply call ``add_global_argument()`` on your +``mach.main.Mach`` instance. e.g.:: + + mach = mach.main.Mach(os.getcwd()) + + # Will allow --example to be specified on every mach command. + mach.add_global_argument('--example', action='store_true', + help='Demonstrate an example global argument.') diff --git a/python/mach/bash-completion.sh b/python/mach/bash-completion.sh new file mode 100644 index 00000000000..e4b151f24c9 --- /dev/null +++ b/python/mach/bash-completion.sh @@ -0,0 +1,29 @@ +function _mach() +{ + local cur cmds c subcommand + COMPREPLY=() + + # Load the list of commands + cmds=`"${COMP_WORDS[0]}" mach-commands` + + # Look for the subcommand. + cur="${COMP_WORDS[COMP_CWORD]}" + subcommand="" + c=1 + while [ $c -lt $COMP_CWORD ]; do + word="${COMP_WORDS[c]}" + for cmd in $cmds; do + if [ "$cmd" = "$word" ]; then + subcommand="$word" + fi + done + c=$((++c)) + done + + if [[ "$subcommand" == "help" || -z "$subcommand" ]]; then + COMPREPLY=( $(compgen -W "$cmds" -- ${cur}) ) + fi + + return 0 +} +complete -o default -F _mach mach diff --git a/python/mach/mach/__init__.py b/python/mach/mach/__init__.py new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/python/mach/mach/__init__.py diff --git a/python/mach/mach/base.py b/python/mach/mach/base.py new file mode 100644 index 00000000000..3e8e6477357 --- /dev/null +++ b/python/mach/mach/base.py @@ -0,0 +1,110 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + + +class CommandContext(object): + """Holds run-time state so it can easily be passed to command providers.""" + def __init__(self, cwd=None, settings=None, log_manager=None, + commands=None, **kwargs): + self.cwd = cwd + self.settings = settings + self.log_manager = log_manager + self.commands = commands + + for k,v in kwargs.items(): + setattr(self, k, v) + + +class MachError(Exception): + """Base class for all errors raised by mach itself.""" + + +class NoCommandError(MachError): + """No command was passed into mach.""" + + +class UnknownCommandError(MachError): + """Raised when we attempted to execute an unknown command.""" + + def __init__(self, command, verb, suggested_commands=None): + MachError.__init__(self) + + self.command = command + self.verb = verb + self.suggested_commands = suggested_commands or [] + +class UnrecognizedArgumentError(MachError): + """Raised when an unknown argument is passed to mach.""" + + def __init__(self, command, arguments): + MachError.__init__(self) + + self.command = command + self.arguments = arguments + + +class MethodHandler(object): + """Describes a Python method that implements a mach command. + + Instances of these are produced by mach when it processes classes + defining mach commands. + """ + __slots__ = ( + # The Python class providing the command. This is the class type not + # an instance of the class. Mach will instantiate a new instance of + # the class if the command is executed. + 'cls', + + # Whether the __init__ method of the class should receive a mach + # context instance. This should only affect the mach driver and how + # it instantiates classes. + 'pass_context', + + # The name of the method providing the command. In other words, this + # is the str name of the attribute on the class type corresponding to + # the name of the function. + 'method', + + # The name of the command. + 'name', + + # String category this command belongs to. + 'category', + + # Description of the purpose of this command. + 'description', + + # Whether to allow all arguments from the parser. + 'allow_all_arguments', + + # Functions used to 'skip' commands if they don't meet the conditions + # in a given context. + 'conditions', + + # argparse.ArgumentParser instance to use as the basis for command + # arguments. + 'parser', + + # Arguments added to this command's parser. This is a 2-tuple of + # positional and named arguments, respectively. + 'arguments', + ) + + def __init__(self, cls, method, name, category=None, description=None, + allow_all_arguments=False, conditions=None, parser=None, arguments=None, + pass_context=False): + + self.cls = cls + self.method = method + self.name = name + self.category = category + self.description = description + self.allow_all_arguments = allow_all_arguments + self.conditions = conditions or [] + self.parser = parser + self.arguments = arguments or [] + self.pass_context = pass_context + diff --git a/python/mach/mach/commands/__init__.py b/python/mach/mach/commands/__init__.py new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/python/mach/mach/commands/__init__.py diff --git a/python/mach/mach/commands/commandinfo.py b/python/mach/mach/commands/commandinfo.py new file mode 100644 index 00000000000..3cca0af202e --- /dev/null +++ b/python/mach/mach/commands/commandinfo.py @@ -0,0 +1,41 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, # You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import print_function, unicode_literals + +from mach.decorators import ( + CommandProvider, + Command, +) + + +@CommandProvider +class BuiltinCommands(object): + def __init__(self, context): + self.context = context + + @Command('mach-commands', category='misc', + description='List all mach commands.') + def commands(self): + print("\n".join(self.context.commands.command_handlers.keys())) + + @Command('mach-debug-commands', category='misc', + description='Show info about available mach commands.') + def debug_commands(self): + import inspect + + handlers = self.context.commands.command_handlers + for command in sorted(handlers.keys()): + handler = handlers[command] + cls = handler.cls + method = getattr(cls, getattr(handler, 'method')) + + print(command) + print('=' * len(command)) + print('') + print('File: %s' % inspect.getsourcefile(method)) + print('Class: %s' % cls.__name__) + print('Method: %s' % handler.method) + print('') + diff --git a/python/mach/mach/commands/settings.py b/python/mach/mach/commands/settings.py new file mode 100644 index 00000000000..7223f241fb2 --- /dev/null +++ b/python/mach/mach/commands/settings.py @@ -0,0 +1,50 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import print_function, unicode_literals + +from textwrap import TextWrapper + +from mach.decorators import ( + CommandProvider, + Command, +) + + +#@CommandProvider +class Settings(object): + """Interact with settings for mach. + + Currently, we only provide functionality to view what settings are + available. In the future, this module will be used to modify settings, help + people create configs via a wizard, etc. + """ + def __init__(self, context): + self.settings = context.settings + + @Command('settings-list', category='devenv', + description='Show available config settings.') + def list_settings(self): + """List available settings in a concise list.""" + for section in sorted(self.settings): + for option in sorted(self.settings[section]): + short, full = self.settings.option_help(section, option) + print('%s.%s -- %s' % (section, option, short)) + + @Command('settings-create', category='devenv', + description='Print a new settings file with usage info.') + def create(self): + """Create an empty settings file with full documentation.""" + wrapper = TextWrapper(initial_indent='# ', subsequent_indent='# ') + + for section in sorted(self.settings): + print('[%s]' % section) + print('') + + for option in sorted(self.settings[section]): + short, full = self.settings.option_help(section, option) + + print(wrapper.fill(full)) + print(';%s =' % option) + print('') diff --git a/python/mach/mach/config.py b/python/mach/mach/config.py new file mode 100644 index 00000000000..89824d554d0 --- /dev/null +++ b/python/mach/mach/config.py @@ -0,0 +1,488 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +r""" +This file defines classes for representing config data/settings. + +Config data is modeled as key-value pairs. Keys are grouped together into named +sections. Individual config settings (options) have metadata associated with +them. This metadata includes type, default value, valid values, etc. + +The main interface to config data is the ConfigSettings class. 1 or more +ConfigProvider classes are associated with ConfigSettings and define what +settings are available. + +Descriptions of individual config options can be translated to multiple +languages using gettext. Each option has associated with it a domain and locale +directory. By default, the domain is the section the option is in and the +locale directory is the "locale" directory beneath the directory containing the +module that defines it. + +People implementing ConfigProvider instances are expected to define a complete +gettext .po and .mo file for the en-US locale. You can use the gettext-provided +msgfmt binary to perform this conversion. Generation of the original .po file +can be done via the write_pot() of ConfigSettings. +""" + +from __future__ import unicode_literals + +import collections +import gettext +import os +import sys + +if sys.version_info[0] == 3: + from configparser import RawConfigParser + str_type = str +else: + from ConfigParser import RawConfigParser + str_type = basestring + + +class ConfigType(object): + """Abstract base class for config values.""" + + @staticmethod + def validate(value): + """Validates a Python value conforms to this type. + + Raises a TypeError or ValueError if it doesn't conform. Does not do + anything if the value is valid. + """ + + @staticmethod + def from_config(config, section, option): + """Obtain the value of this type from a RawConfigParser. + + Receives a RawConfigParser instance, a str section name, and the str + option in that section to retrieve. + + The implementation may assume the option exists in the RawConfigParser + instance. + + Implementations are not expected to validate the value. But, they + should return the appropriate Python type. + """ + + @staticmethod + def to_config(value): + return value + + +class StringType(ConfigType): + @staticmethod + def validate(value): + if not isinstance(value, str_type): + raise TypeError() + + @staticmethod + def from_config(config, section, option): + return config.get(section, option) + + +class BooleanType(ConfigType): + @staticmethod + def validate(value): + if not isinstance(value, bool): + raise TypeError() + + @staticmethod + def from_config(config, section, option): + return config.getboolean(section, option) + + @staticmethod + def to_config(value): + return 'true' if value else 'false' + + +class IntegerType(ConfigType): + @staticmethod + def validate(value): + if not isinstance(value, int): + raise TypeError() + + @staticmethod + def from_config(config, section, option): + return config.getint(section, option) + + +class PositiveIntegerType(IntegerType): + @staticmethod + def validate(value): + if not isinstance(value, int): + raise TypeError() + + if value < 0: + raise ValueError() + + +class PathType(StringType): + @staticmethod + def validate(value): + if not isinstance(value, str_type): + raise TypeError() + + @staticmethod + def from_config(config, section, option): + return config.get(section, option) + + +class AbsolutePathType(PathType): + @staticmethod + def validate(value): + if not isinstance(value, str_type): + raise TypeError() + + if not os.path.isabs(value): + raise ValueError() + + +class RelativePathType(PathType): + @staticmethod + def validate(value): + if not isinstance(value, str_type): + raise TypeError() + + if os.path.isabs(value): + raise ValueError() + + +class DefaultValue(object): + pass + + +class ConfigProvider(object): + """Abstract base class for an object providing config settings. + + Classes implementing this interface expose configurable settings. Settings + are typically only relevant to that component itself. But, nothing says + settings can't be shared by multiple components. + """ + + @classmethod + def register_settings(cls): + """Registers config settings. + + This is called automatically. Child classes should likely not touch it. + See _register_settings() instead. + """ + if hasattr(cls, '_settings_registered'): + return + + cls._settings_registered = True + + cls.config_settings = {} + + ourdir = os.path.dirname(__file__) + cls.config_settings_locale_directory = os.path.join(ourdir, 'locale') + + cls._register_settings() + + @classmethod + def _register_settings(cls): + """The actual implementation of register_settings(). + + This is what child classes should implement. They should not touch + register_settings(). + + Implementations typically make 1 or more calls to _register_setting(). + """ + raise NotImplemented('%s must implement _register_settings.' % + __name__) + + @classmethod + def register_setting(cls, section, option, type_cls, default=DefaultValue, + choices=None, domain=None): + """Register a config setting with this type. + + This is a convenience method to populate available settings. It is + typically called in the class's _register_settings() implementation. + + Each setting must have: + + section -- str section to which the setting belongs. This is how + settings are grouped. + + option -- str id for the setting. This must be unique within the + section it appears. + + type -- a ConfigType-derived type defining the type of the setting. + + Each setting has the following optional parameters: + + default -- The default value for the setting. If None (the default) + there is no default. + + choices -- A set of values this setting can hold. Values not in + this set are invalid. + + domain -- Translation domain for this setting. By default, the + domain is the same as the section name. + """ + if not section in cls.config_settings: + cls.config_settings[section] = {} + + if option in cls.config_settings[section]: + raise Exception('Setting has already been registered: %s.%s' % ( + section, option)) + + domain = domain if domain is not None else section + + meta = { + 'short': '%s.short' % option, + 'full': '%s.full' % option, + 'type_cls': type_cls, + 'domain': domain, + 'localedir': cls.config_settings_locale_directory, + } + + if default != DefaultValue: + meta['default'] = default + + if choices is not None: + meta['choices'] = choices + + cls.config_settings[section][option] = meta + + +class ConfigSettings(collections.Mapping): + """Interface for configuration settings. + + This is the main interface to the configuration. + + A configuration is a collection of sections. Each section contains + key-value pairs. + + When an instance is created, the caller first registers ConfigProvider + instances with it. This tells the ConfigSettings what individual settings + are available and defines extra metadata associated with those settings. + This is used for validation, etc. + + Once ConfigProvider instances are registered, a config is populated. It can + be loaded from files or populated by hand. + + ConfigSettings instances are accessed like dictionaries or by using + attributes. e.g. the section "foo" is accessed through either + settings.foo or settings['foo']. + + Sections are modeled by the ConfigSection class which is defined inside + this one. They look just like dicts or classes with attributes. To access + the "bar" option in the "foo" section: + + value = settings.foo.bar + value = settings['foo']['bar'] + value = settings.foo['bar'] + + Assignment is similar: + + settings.foo.bar = value + settings['foo']['bar'] = value + settings['foo'].bar = value + + You can even delete user-assigned values: + + del settings.foo.bar + del settings['foo']['bar'] + + If there is a default, it will be returned. + + When settings are mutated, they are validated against the registered + providers. Setting unknown settings or setting values to illegal values + will result in exceptions being raised. + """ + + class ConfigSection(collections.MutableMapping, object): + """Represents an individual config section.""" + def __init__(self, config, name, settings): + object.__setattr__(self, '_config', config) + object.__setattr__(self, '_name', name) + object.__setattr__(self, '_settings', settings) + + # MutableMapping interface + def __len__(self): + return len(self._settings) + + def __iter__(self): + return iter(self._settings.keys()) + + def __contains__(self, k): + return k in self._settings + + def __getitem__(self, k): + if k not in self._settings: + raise KeyError('Option not registered with provider: %s' % k) + + meta = self._settings[k] + + if self._config.has_option(self._name, k): + return meta['type_cls'].from_config(self._config, self._name, k) + + if not 'default' in meta: + raise KeyError('No default value registered: %s' % k) + + return meta['default'] + + def __setitem__(self, k, v): + if k not in self._settings: + raise KeyError('Option not registered with provider: %s' % k) + + meta = self._settings[k] + + meta['type_cls'].validate(v) + + if not self._config.has_section(self._name): + self._config.add_section(self._name) + + self._config.set(self._name, k, meta['type_cls'].to_config(v)) + + def __delitem__(self, k): + self._config.remove_option(self._name, k) + + # Prune empty sections. + if not len(self._config.options(self._name)): + self._config.remove_section(self._name) + + def __getattr__(self, k): + return self.__getitem__(k) + + def __setattr__(self, k, v): + self.__setitem__(k, v) + + def __delattr__(self, k): + self.__delitem__(k) + + + def __init__(self): + self._config = RawConfigParser() + + self._settings = {} + self._sections = {} + self._finalized = False + self._loaded_filenames = set() + + def load_file(self, filename): + self.load_files([filename]) + + def load_files(self, filenames): + """Load a config from files specified by their paths. + + Files are loaded in the order given. Subsequent files will overwrite + values from previous files. If a file does not exist, it will be + ignored. + """ + filtered = [f for f in filenames if os.path.exists(f)] + + fps = [open(f, 'rt') for f in filtered] + self.load_fps(fps) + self._loaded_filenames.update(set(filtered)) + for fp in fps: + fp.close() + + def load_fps(self, fps): + """Load config data by reading file objects.""" + + for fp in fps: + self._config.readfp(fp) + + def loaded_files(self): + return self._loaded_filenames + + def write(self, fh): + """Write the config to a file object.""" + self._config.write(fh) + + def validate(self): + """Ensure that the current config passes validation. + + This is a generator of tuples describing any validation errors. The + elements of the tuple are: + + (bool) True if error is fatal. False if just a warning. + (str) Type of validation issue. Can be one of ('unknown-section', + 'missing-required', 'type-error') + """ + + def register_provider(self, provider): + """Register a ConfigProvider with this settings interface.""" + + if self._finalized: + raise Exception('Providers cannot be registered after finalized.') + + provider.register_settings() + + for section_name, settings in provider.config_settings.items(): + section = self._settings.get(section_name, {}) + + for k, v in settings.items(): + if k in section: + raise Exception('Setting already registered: %s.%s' % + section_name, k) + + section[k] = v + + self._settings[section_name] = section + + def write_pot(self, fh): + """Write a pot gettext translation file.""" + + for section in sorted(self): + fh.write('# Section %s\n\n' % section) + for option in sorted(self[section]): + fh.write('msgid "%s.%s.short"\n' % (section, option)) + fh.write('msgstr ""\n\n') + + fh.write('msgid "%s.%s.full"\n' % (section, option)) + fh.write('msgstr ""\n\n') + + fh.write('# End of section %s\n\n' % section) + + def option_help(self, section, option): + """Obtain the translated help messages for an option.""" + + meta = self[section]._settings[option] + + # Providers should always have an en-US translation. If they don't, + # they are coded wrong and this will raise. + default = gettext.translation(meta['domain'], meta['localedir'], + ['en-US']) + + t = gettext.translation(meta['domain'], meta['localedir'], + fallback=True) + t.add_fallback(default) + + short = t.ugettext('%s.%s.short' % (section, option)) + full = t.ugettext('%s.%s.full' % (section, option)) + + return (short, full) + + def _finalize(self): + if self._finalized: + return + + for section, settings in self._settings.items(): + s = ConfigSettings.ConfigSection(self._config, section, settings) + self._sections[section] = s + + self._finalized = True + + # Mapping interface. + def __len__(self): + return len(self._settings) + + def __iter__(self): + self._finalize() + + return iter(self._sections.keys()) + + def __contains__(self, k): + return k in self._settings + + def __getitem__(self, k): + self._finalize() + + return self._sections[k] + + # Allow attribute access because it looks nice. + def __getattr__(self, k): + return self.__getitem__(k) diff --git a/python/mach/mach/decorators.py b/python/mach/mach/decorators.py new file mode 100644 index 00000000000..cb9a67428e7 --- /dev/null +++ b/python/mach/mach/decorators.py @@ -0,0 +1,176 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import collections +import inspect +import types + +from .base import ( + MachError, + MethodHandler +) + +from .config import ConfigProvider +from .registrar import Registrar + + +def CommandProvider(cls): + """Class decorator to denote that it provides subcommands for Mach. + + When this decorator is present, mach looks for commands being defined by + methods inside the class. + """ + + # The implementation of this decorator relies on the parse-time behavior of + # decorators. When the module is imported, the method decorators (like + # @Command and @CommandArgument) are called *before* this class decorator. + # The side-effect of the method decorators is to store specifically-named + # attributes on the function types. We just scan over all functions in the + # class looking for the side-effects of the method decorators. + + # Tell mach driver whether to pass context argument to __init__. + pass_context = False + + if inspect.ismethod(cls.__init__): + spec = inspect.getargspec(cls.__init__) + + if len(spec.args) > 2: + msg = 'Mach @CommandProvider class %s implemented incorrectly. ' + \ + '__init__() must take 1 or 2 arguments. From %s' + msg = msg % (cls.__name__, inspect.getsourcefile(cls)) + raise MachError(msg) + + if len(spec.args) == 2: + pass_context = True + + # We scan __dict__ because we only care about the classes own attributes, + # not inherited ones. If we did inherited attributes, we could potentially + # define commands multiple times. We also sort keys so commands defined in + # the same class are grouped in a sane order. + for attr in sorted(cls.__dict__.keys()): + value = cls.__dict__[attr] + + if not isinstance(value, types.FunctionType): + continue + + command_name, category, description, allow_all, conditions, parser = getattr( + value, '_mach_command', (None, None, None, None, None, None)) + + if command_name is None: + continue + + if conditions is None and Registrar.require_conditions: + continue + + msg = 'Mach command \'%s\' implemented incorrectly. ' + \ + 'Conditions argument must take a list ' + \ + 'of functions. Found %s instead.' + + conditions = conditions or [] + if not isinstance(conditions, collections.Iterable): + msg = msg % (command_name, type(conditions)) + raise MachError(msg) + + for c in conditions: + if not hasattr(c, '__call__'): + msg = msg % (command_name, type(c)) + raise MachError(msg) + + arguments = getattr(value, '_mach_command_args', None) + + handler = MethodHandler(cls, attr, command_name, category=category, + description=description, allow_all_arguments=allow_all, + conditions=conditions, parser=parser, arguments=arguments, + pass_context=pass_context) + + Registrar.register_command_handler(handler) + + return cls + + +class Command(object): + """Decorator for functions or methods that provide a mach subcommand. + + The decorator accepts arguments that define basic attributes of the + command. The following arguments are recognized: + + category -- The string category to which this command belongs. Mach's + help will group commands by category. + + description -- A brief description of what the command does. + + allow_all_args -- Bool indicating whether to allow unknown arguments + through to the command. + + parser -- an optional argparse.ArgumentParser instance to use as + the basis for the command arguments. + + For example: + + @Command('foo', category='misc', description='Run the foo action') + def foo(self): + pass + """ + def __init__(self, name, category=None, description=None, + allow_all_args=False, conditions=None, parser=None): + self._name = name + self._category = category + self._description = description + self._allow_all_args = allow_all_args + self._conditions = conditions + self._parser = parser + + def __call__(self, func): + func._mach_command = (self._name, self._category, self._description, + self._allow_all_args, self._conditions, self._parser) + + return func + + +class CommandArgument(object): + """Decorator for additional arguments to mach subcommands. + + This decorator should be used to add arguments to mach commands. Arguments + to the decorator are proxied to ArgumentParser.add_argument(). + + For example: + + @Command('foo', help='Run the foo action') + @CommandArgument('-b', '--bar', action='store_true', default=False, + help='Enable bar mode.') + def foo(self): + pass + """ + def __init__(self, *args, **kwargs): + self._command_args = (args, kwargs) + + def __call__(self, func): + command_args = getattr(func, '_mach_command_args', []) + + command_args.insert(0, self._command_args) + + func._mach_command_args = command_args + + return func + + +def SettingsProvider(cls): + """Class decorator to denote that this class provides Mach settings. + + When this decorator is encountered, the underlying class will automatically + be registered with the Mach registrar and will (likely) be hooked up to the + mach driver. + + This decorator is only allowed on mach.config.ConfigProvider classes. + """ + if not issubclass(cls, ConfigProvider): + raise MachError('@SettingsProvider encountered on class that does ' + + 'not derived from mach.config.ConfigProvider.') + + Registrar.register_settings_provider(cls) + + return cls + diff --git a/python/mach/mach/dispatcher.py b/python/mach/mach/dispatcher.py new file mode 100644 index 00000000000..11e58cb9e2f --- /dev/null +++ b/python/mach/mach/dispatcher.py @@ -0,0 +1,277 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import argparse +import difflib +import sys + +from operator import itemgetter + +from .base import ( + NoCommandError, + UnknownCommandError, + UnrecognizedArgumentError, +) + + +class CommandFormatter(argparse.HelpFormatter): + """Custom formatter to format just a subcommand.""" + + def add_usage(self, *args): + pass + + +class CommandAction(argparse.Action): + """An argparse action that handles mach commands. + + This class is essentially a reimplementation of argparse's sub-parsers + feature. We first tried to use sub-parsers. However, they were missing + features like grouping of commands (http://bugs.python.org/issue14037). + + The way this works involves light magic and a partial understanding of how + argparse works. + + Arguments registered with an argparse.ArgumentParser have an action + associated with them. An action is essentially a class that when called + does something with the encountered argument(s). This class is one of those + action classes. + + An instance of this class is created doing something like: + + parser.add_argument('command', action=CommandAction, registrar=r) + + Note that a mach.registrar.Registrar instance is passed in. The Registrar + holds information on all the mach commands that have been registered. + + When this argument is registered with the ArgumentParser, an instance of + this class is instantiated. One of the subtle but important things it does + is tell the argument parser that it's interested in *all* of the remaining + program arguments. So, when the ArgumentParser calls this action, we will + receive the command name plus all of its arguments. + + For more, read the docs in __call__. + """ + def __init__(self, option_strings, dest, required=True, default=None, + registrar=None, context=None): + # A proper API would have **kwargs here. However, since we are a little + # hacky, we intentionally omit it as a way of detecting potentially + # breaking changes with argparse's implementation. + # + # In a similar vein, default is passed in but is not needed, so we drop + # it. + argparse.Action.__init__(self, option_strings, dest, required=required, + help=argparse.SUPPRESS, nargs=argparse.REMAINDER) + + self._mach_registrar = registrar + self._context = context + + def __call__(self, parser, namespace, values, option_string=None): + """This is called when the ArgumentParser has reached our arguments. + + Since we always register ourselves with nargs=argparse.REMAINDER, + values should be a list of remaining arguments to parse. The first + argument should be the name of the command to invoke and all remaining + arguments are arguments for that command. + + The gist of the flow is that we look at the command being invoked. If + it's *help*, we handle that specially (because argparse's default help + handler isn't satisfactory). Else, we create a new, independent + ArgumentParser instance for just the invoked command (based on the + information contained in the command registrar) and feed the arguments + into that parser. We then merge the results with the main + ArgumentParser. + """ + if namespace.help: + # -h or --help is in the global arguments. + self._handle_main_help(parser, namespace.verbose) + sys.exit(0) + elif values: + command = values[0].lower() + args = values[1:] + + if command == 'help': + if args and args[0] not in ['-h', '--help']: + # Make sure args[0] is indeed a command. + self._handle_subcommand_help(parser, args[0]) + else: + self._handle_main_help(parser, namespace.verbose) + sys.exit(0) + elif '-h' in args or '--help' in args: + # -h or --help is in the command arguments. + self._handle_subcommand_help(parser, command) + sys.exit(0) + else: + raise NoCommandError() + + # Command suggestion + if command not in self._mach_registrar.command_handlers: + # We first try to look for a valid command that is very similar to the given command. + suggested_commands = difflib.get_close_matches(command, self._mach_registrar.command_handlers.keys(), cutoff=0.8) + # If we find more than one matching command, or no command at all, we give command suggestions instead + # (with a lower matching threshold). All commands that start with the given command (for instance: 'mochitest-plain', + # 'mochitest-chrome', etc. for 'mochitest-') are also included. + if len(suggested_commands) != 1: + suggested_commands = set(difflib.get_close_matches(command, self._mach_registrar.command_handlers.keys(), cutoff=0.5)) + suggested_commands |= {cmd for cmd in self._mach_registrar.command_handlers if cmd.startswith(command)} + raise UnknownCommandError(command, 'run', suggested_commands) + sys.stderr.write("We're assuming the '%s' command is '%s' and we're executing it for you.\n\n" % (command, suggested_commands[0])) + command = suggested_commands[0] + + handler = self._mach_registrar.command_handlers.get(command) + + # FUTURE + # If we wanted to conditionally enable commands based on whether + # it's possible to run them given the current state of system, here + # would be a good place to hook that up. + + # We create a new parser, populate it with the command's arguments, + # then feed all remaining arguments to it, merging the results + # with ourselves. This is essentially what argparse subparsers + # do. + + parser_args = { + 'add_help': False, + 'usage': '%(prog)s [global arguments] ' + command + + ' command arguments]', + } + + if handler.allow_all_arguments: + parser_args['prefix_chars'] = '+' + + if handler.parser: + subparser = handler.parser + else: + subparser = argparse.ArgumentParser(**parser_args) + + for arg in handler.arguments: + subparser.add_argument(*arg[0], **arg[1]) + + # We define the command information on the main parser result so as to + # not interfere with arguments passed to the command. + setattr(namespace, 'mach_handler', handler) + setattr(namespace, 'command', command) + + command_namespace, extra = subparser.parse_known_args(args) + setattr(namespace, 'command_args', command_namespace) + if extra: + raise UnrecognizedArgumentError(command, extra) + + def _handle_main_help(self, parser, verbose): + # Since we don't need full sub-parser support for the main help output, + # we create groups in the ArgumentParser and populate each group with + # arguments corresponding to command names. This has the side-effect + # that argparse renders it nicely. + r = self._mach_registrar + disabled_commands = [] + + cats = [(k, v[2]) for k, v in r.categories.items()] + sorted_cats = sorted(cats, key=itemgetter(1), reverse=True) + for category, priority in sorted_cats: + group = None + + for command in sorted(r.commands_by_category[category]): + handler = r.command_handlers[command] + + # Instantiate a handler class to see if it should be filtered + # out for the current context or not. Condition functions can be + # applied to the command's decorator. + if handler.conditions: + if handler.pass_context: + instance = handler.cls(self._context) + else: + instance = handler.cls() + + is_filtered = False + for c in handler.conditions: + if not c(instance): + is_filtered = True + break + if is_filtered: + description = handler.description + disabled_command = {'command': command, 'description': description} + disabled_commands.append(disabled_command) + continue + + if group is None: + title, description, _priority = r.categories[category] + group = parser.add_argument_group(title, description) + + description = handler.description + group.add_argument(command, help=description, + action='store_true') + + if disabled_commands and 'disabled' in r.categories: + title, description, _priority = r.categories['disabled'] + group = parser.add_argument_group(title, description) + if verbose == True: + for c in disabled_commands: + group.add_argument(c['command'], help=c['description'], + action='store_true') + + parser.print_help() + + def _handle_subcommand_help(self, parser, command): + handler = self._mach_registrar.command_handlers.get(command) + + if not handler: + raise UnknownCommandError(command, 'query') + + # This code is worth explaining. Because we are doing funky things with + # argument registration to allow the same option in both global and + # command arguments, we can't simply put all arguments on the same + # parser instance because argparse would complain. We can't register an + # argparse subparser here because it won't properly show help for + # global arguments. So, we employ a strategy similar to command + # execution where we construct a 2nd, independent ArgumentParser for + # just the command data then supplement the main help's output with + # this 2nd parser's. We use a custom formatter class to ignore some of + # the help output. + parser_args = { + 'formatter_class': CommandFormatter, + 'add_help': False, + } + + if handler.allow_all_arguments: + parser_args['prefix_chars'] = '+' + + if handler.parser: + c_parser = handler.parser + c_parser.formatter_class = NoUsageFormatter + # Accessing _action_groups is a bit shady. We are highly dependent + # on the argparse implementation not changing. We fail fast to + # detect upstream changes so we can intelligently react to them. + group = c_parser._action_groups[1] + + # By default argparse adds two groups called "positional arguments" + # and "optional arguments". We want to rename these to reflect standard + # mach terminology. + c_parser._action_groups[0].title = 'Command Parameters' + c_parser._action_groups[1].title = 'Command Arguments' + + if not handler.description: + handler.description = c_parser.description + c_parser.description = None + else: + c_parser = argparse.ArgumentParser(**parser_args) + group = c_parser.add_argument_group('Command Arguments') + + for arg in handler.arguments: + group.add_argument(*arg[0], **arg[1]) + + # This will print the description of the command below the usage. + description = handler.description + if description: + parser.description = description + + parser.usage = '%(prog)s [global arguments] ' + command + \ + ' [command arguments]' + parser.print_help() + print('') + c_parser.print_help() + +class NoUsageFormatter(argparse.HelpFormatter): + def _format_usage(self, *args, **kwargs): + return "" diff --git a/python/mach/mach/logging.py b/python/mach/mach/logging.py new file mode 100644 index 00000000000..729e6cb3d93 --- /dev/null +++ b/python/mach/mach/logging.py @@ -0,0 +1,256 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +# This file contains logging functionality for mach. It essentially provides +# support for a structured logging framework built on top of Python's built-in +# logging framework. + +from __future__ import absolute_import, unicode_literals + +try: + import blessings +except ImportError: + blessings = None + +import json +import logging +import sys +import time + + +def format_seconds(total): + """Format number of seconds to MM:SS.DD form.""" + + minutes, seconds = divmod(total, 60) + + return '%2d:%05.2f' % (minutes, seconds) + + +class ConvertToStructuredFilter(logging.Filter): + """Filter that converts unstructured records into structured ones.""" + def filter(self, record): + if hasattr(record, 'action') and hasattr(record, 'params'): + return True + + record.action = 'unstructured' + record.params = {'msg': record.getMessage()} + record.msg = '{msg}' + + return True + + +class StructuredJSONFormatter(logging.Formatter): + """Log formatter that writes a structured JSON entry.""" + + def format(self, record): + action = getattr(record, 'action', 'UNKNOWN') + params = getattr(record, 'params', {}) + + return json.dumps([record.created, action, params]) + + +class StructuredHumanFormatter(logging.Formatter): + """Log formatter that writes structured messages for humans. + + It is important that this formatter never be added to a logger that + produces unstructured/classic log messages. If it is, the call to format() + could fail because the string could contain things (like JSON) that look + like formatting character sequences. + + Because of this limitation, format() will fail with a KeyError if an + unstructured record is passed or if the structured message is malformed. + """ + def __init__(self, start_time, write_interval=False, write_times=True): + self.start_time = start_time + self.write_interval = write_interval + self.write_times = write_times + self.last_time = None + + def format(self, record): + f = record.msg.format(**record.params) + + if not self.write_times: + return f + + elapsed = self._time(record) + + return '%s %s' % (format_seconds(elapsed), f) + + def _time(self, record): + t = record.created - self.start_time + + if self.write_interval and self.last_time is not None: + t = record.created - self.last_time + + self.last_time = record.created + + return t + + +class StructuredTerminalFormatter(StructuredHumanFormatter): + """Log formatter for structured messages writing to a terminal.""" + + def set_terminal(self, terminal): + self.terminal = terminal + + def format(self, record): + f = record.msg.format(**record.params) + + if not self.write_times: + return f + + t = self.terminal.blue(format_seconds(self._time(record))) + + return '%s %s' % (t, self._colorize(f)) + + def _colorize(self, s): + if not self.terminal: + return s + + result = s + + reftest = s.startswith('REFTEST ') + if reftest: + s = s[8:] + + if s.startswith('TEST-PASS'): + result = self.terminal.green(s[0:9]) + s[9:] + elif s.startswith('TEST-UNEXPECTED'): + result = self.terminal.red(s[0:20]) + s[20:] + elif s.startswith('TEST-START'): + result = self.terminal.yellow(s[0:10]) + s[10:] + elif s.startswith('TEST-INFO'): + result = self.terminal.yellow(s[0:9]) + s[9:] + + if reftest: + result = 'REFTEST ' + result + + return result + + +class LoggingManager(object): + """Holds and controls global logging state. + + An application should instantiate one of these and configure it as needed. + + This class provides a mechanism to configure the output of logging data + both from mach and from the overall logging system (e.g. from other + modules). + """ + + def __init__(self): + self.start_time = time.time() + + self.json_handlers = [] + self.terminal_handler = None + self.terminal_formatter = None + + self.root_logger = logging.getLogger() + self.root_logger.setLevel(logging.DEBUG) + + # Installing NullHandler on the root logger ensures that *all* log + # messages have at least one handler. This prevents Python from + # complaining about "no handlers could be found for logger XXX." + self.root_logger.addHandler(logging.NullHandler()) + + self.mach_logger = logging.getLogger('mach') + self.mach_logger.setLevel(logging.DEBUG) + + self.structured_filter = ConvertToStructuredFilter() + + self.structured_loggers = [self.mach_logger] + + self._terminal = None + + @property + def terminal(self): + if not self._terminal and blessings: + # Sometimes blessings fails to set up the terminal. In that case, + # silently fail. + try: + terminal = blessings.Terminal(stream=sys.stdout) + + if terminal.is_a_tty: + self._terminal = terminal + except Exception: + pass + + return self._terminal + + def add_json_handler(self, fh): + """Enable JSON logging on the specified file object.""" + + # Configure the consumer of structured messages. + handler = logging.StreamHandler(stream=fh) + handler.setFormatter(StructuredJSONFormatter()) + handler.setLevel(logging.DEBUG) + + # And hook it up. + for logger in self.structured_loggers: + logger.addHandler(handler) + + self.json_handlers.append(handler) + + def add_terminal_logging(self, fh=sys.stdout, level=logging.INFO, + write_interval=False, write_times=True): + """Enable logging to the terminal.""" + + formatter = StructuredHumanFormatter(self.start_time, + write_interval=write_interval, write_times=write_times) + + if self.terminal: + formatter = StructuredTerminalFormatter(self.start_time, + write_interval=write_interval, write_times=write_times) + formatter.set_terminal(self.terminal) + + handler = logging.StreamHandler(stream=fh) + handler.setFormatter(formatter) + handler.setLevel(level) + + for logger in self.structured_loggers: + logger.addHandler(handler) + + self.terminal_handler = handler + self.terminal_formatter = formatter + + def replace_terminal_handler(self, handler): + """Replace the installed terminal handler. + + Returns the old handler or None if none was configured. + If the new handler is None, removes any existing handler and disables + logging to the terminal. + """ + old = self.terminal_handler + + if old: + for logger in self.structured_loggers: + logger.removeHandler(old) + + if handler: + for logger in self.structured_loggers: + logger.addHandler(handler) + + self.terminal_handler = handler + + return old + + def enable_unstructured(self): + """Enable logging of unstructured messages.""" + if self.terminal_handler: + self.terminal_handler.addFilter(self.structured_filter) + self.root_logger.addHandler(self.terminal_handler) + + def disable_unstructured(self): + """Disable logging of unstructured messages.""" + if self.terminal_handler: + self.terminal_handler.removeFilter(self.structured_filter) + self.root_logger.removeHandler(self.terminal_handler) + + def register_structured_logger(self, logger): + """Register a structured logger. + + This needs to be called for all structured loggers that don't chain up + to the mach logger in order for their output to be captured. + """ + self.structured_loggers.append(logger) diff --git a/python/mach/mach/main.py b/python/mach/mach/main.py new file mode 100644 index 00000000000..3c74bd51333 --- /dev/null +++ b/python/mach/mach/main.py @@ -0,0 +1,615 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This module provides functionality for the command-line build tool +# (mach). It is packaged as a module because everything is a library. + +from __future__ import absolute_import, print_function, unicode_literals +from collections import Iterable + +import argparse +import codecs +import imp +import logging +import os +import sys +import traceback +import uuid +import sys + +from .base import ( + CommandContext, + MachError, + NoCommandError, + UnknownCommandError, + UnrecognizedArgumentError, +) + +from .decorators import ( + CommandArgument, + CommandProvider, + Command, +) + +from .config import ConfigSettings +from .dispatcher import CommandAction +from .logging import LoggingManager +from .registrar import Registrar + + + +MACH_ERROR = r''' +The error occurred in mach itself. This is likely a bug in mach itself or a +fundamental problem with a loaded module. + +Please consider filing a bug against mach by going to the URL: + + https://bugzilla.mozilla.org/enter_bug.cgi?product=Core&component=mach + +'''.lstrip() + +ERROR_FOOTER = r''' +If filing a bug, please include the full output of mach, including this error +message. + +The details of the failure are as follows: +'''.lstrip() + +COMMAND_ERROR = r''' +The error occurred in the implementation of the invoked mach command. + +This should never occur and is likely a bug in the implementation of that +command. Consider filing a bug for this issue. +'''.lstrip() + +MODULE_ERROR = r''' +The error occurred in code that was called by the mach command. This is either +a bug in the called code itself or in the way that mach is calling it. + +You should consider filing a bug for this issue. +'''.lstrip() + +NO_COMMAND_ERROR = r''' +It looks like you tried to run mach without a command. + +Run |mach help| to show a list of commands. +'''.lstrip() + +UNKNOWN_COMMAND_ERROR = r''' +It looks like you are trying to %s an unknown mach command: %s +%s +Run |mach help| to show a list of commands. +'''.lstrip() + +SUGGESTED_COMMANDS_MESSAGE = r''' +Did you want to %s any of these commands instead: %s? +''' + +UNRECOGNIZED_ARGUMENT_ERROR = r''' +It looks like you passed an unrecognized argument into mach. + +The %s command does not accept the arguments: %s +'''.lstrip() + +INVALID_COMMAND_CONTEXT = r''' +It looks like you tried to run a mach command from an invalid context. The %s +command failed to meet the following conditions: %s + +Run |mach help| to show a list of all commands available to the current context. +'''.lstrip() + +INVALID_ENTRY_POINT = r''' +Entry points should return a list of command providers or directories +containing command providers. The following entry point is invalid: + + %s + +You are seeing this because there is an error in an external module attempting +to implement a mach command. Please fix the error, or uninstall the module from +your system. +'''.lstrip() + +class ArgumentParser(argparse.ArgumentParser): + """Custom implementation argument parser to make things look pretty.""" + + def error(self, message): + """Custom error reporter to give more helpful text on bad commands.""" + if not message.startswith('argument command: invalid choice'): + argparse.ArgumentParser.error(self, message) + assert False + + print('Invalid command specified. The list of commands is below.\n') + self.print_help() + sys.exit(1) + + def format_help(self): + text = argparse.ArgumentParser.format_help(self) + + # Strip out the silly command list that would preceed the pretty list. + # + # Commands: + # {foo,bar} + # foo Do foo. + # bar Do bar. + search = 'Commands:\n {' + start = text.find(search) + + if start != -1: + end = text.find('}\n', start) + assert end != -1 + + real_start = start + len('Commands:\n') + real_end = end + len('}\n') + + text = text[0:real_start] + text[real_end:] + + return text + + +class ContextWrapper(object): + def __init__(self, context, handler): + object.__setattr__(self, '_context', context) + object.__setattr__(self, '_handler', handler) + + def __getattribute__(self, key): + try: + return getattr(object.__getattribute__(self, '_context'), key) + except AttributeError as e: + try: + ret = object.__getattribute__(self, '_handler')(self, key) + except AttributeError, TypeError: + # TypeError is in case the handler comes from old code not + # taking a key argument. + raise e + setattr(self, key, ret) + return ret + + def __setattr__(self, key, value): + setattr(object.__getattribute__(self, '_context'), key, value) + + +@CommandProvider +class Mach(object): + """Main mach driver type. + + This type is responsible for holding global mach state and dispatching + a command from arguments. + + The following attributes may be assigned to the instance to influence + behavior: + + populate_context_handler -- If defined, it must be a callable. The + callable signature is the following: + populate_context_handler(context, key=None) + It acts as a fallback getter for the mach.base.CommandContext + instance. + This allows to augment the context instance with arbitrary data + for use in command handlers. + For backwards compatibility, it is also called before command + dispatch without a key, allowing the context handler to add + attributes to the context instance. + + require_conditions -- If True, commands that do not have any condition + functions applied will be skipped. Defaults to False. + + """ + + USAGE = """%(prog)s [global arguments] command [command arguments] + +mach (German for "do") is the main interface to the Mozilla build system and +common developer tasks. + +You tell mach the command you want to perform and it does it for you. + +Some common commands are: + + %(prog)s build Build/compile the source tree. + %(prog)s help Show full help, including the list of all commands. + +To see more help for a specific command, run: + + %(prog)s help <command> +""" + + def __init__(self, cwd): + assert os.path.isdir(cwd) + + self.cwd = cwd + self.log_manager = LoggingManager() + self.logger = logging.getLogger(__name__) + self.settings = ConfigSettings() + + self.log_manager.register_structured_logger(self.logger) + self.global_arguments = [] + self.populate_context_handler = None + + def add_global_argument(self, *args, **kwargs): + """Register a global argument with the argument parser. + + Arguments are proxied to ArgumentParser.add_argument() + """ + + self.global_arguments.append((args, kwargs)) + + def load_commands_from_directory(self, path): + """Scan for mach commands from modules in a directory. + + This takes a path to a directory, loads the .py files in it, and + registers and found mach command providers with this mach instance. + """ + for f in sorted(os.listdir(path)): + if not f.endswith('.py') or f == '__init__.py': + continue + + full_path = os.path.join(path, f) + module_name = 'mach.commands.%s' % f[0:-3] + + self.load_commands_from_file(full_path, module_name=module_name) + + def load_commands_from_file(self, path, module_name=None): + """Scan for mach commands from a file. + + This takes a path to a file and loads it as a Python module under the + module name specified. If no name is specified, a random one will be + chosen. + """ + if module_name is None: + # Ensure parent module is present otherwise we'll (likely) get + # an error due to unknown parent. + if b'mach.commands' not in sys.modules: + mod = imp.new_module(b'mach.commands') + sys.modules[b'mach.commands'] = mod + + module_name = 'mach.commands.%s' % uuid.uuid1().get_hex() + + imp.load_source(module_name, path) + + def load_commands_from_entry_point(self, group='mach.providers'): + """Scan installed packages for mach command provider entry points. An + entry point is a function that returns a list of paths to files or + directories containing command providers. + + This takes an optional group argument which specifies the entry point + group to use. If not specified, it defaults to 'mach.providers'. + """ + try: + import pkg_resources + except ImportError: + print("Could not find setuptools, ignoring command entry points", + file=sys.stderr) + return + + for entry in pkg_resources.iter_entry_points(group=group, name=None): + paths = entry.load()() + if not isinstance(paths, Iterable): + print(INVALID_ENTRY_POINT % entry) + sys.exit(1) + + for path in paths: + if os.path.isfile(path): + self.load_commands_from_file(path) + elif os.path.isdir(path): + self.load_commands_from_directory(path) + else: + print("command provider '%s' does not exist" % path) + + def define_category(self, name, title, description, priority=50): + """Provide a description for a named command category.""" + + Registrar.register_category(name, title, description, priority) + + @property + def require_conditions(self): + return Registrar.require_conditions + + @require_conditions.setter + def require_conditions(self, value): + Registrar.require_conditions = value + + def run(self, argv, stdin=None, stdout=None, stderr=None): + """Runs mach with arguments provided from the command line. + + Returns the integer exit code that should be used. 0 means success. All + other values indicate failure. + """ + + # If no encoding is defined, we default to UTF-8 because without this + # Python 2.7 will assume the default encoding of ASCII. This will blow + # up with UnicodeEncodeError as soon as it encounters a non-ASCII + # character in a unicode instance. We simply install a wrapper around + # the streams and restore once we have finished. + stdin = sys.stdin if stdin is None else stdin + stdout = sys.stdout if stdout is None else stdout + stderr = sys.stderr if stderr is None else stderr + + orig_stdin = sys.stdin + orig_stdout = sys.stdout + orig_stderr = sys.stderr + + sys.stdin = stdin + sys.stdout = stdout + sys.stderr = stderr + + try: + if stdin.encoding is None: + sys.stdin = codecs.getreader('utf-8')(stdin) + + if stdout.encoding is None: + sys.stdout = codecs.getwriter('utf-8')(stdout) + + if stderr.encoding is None: + sys.stderr = codecs.getwriter('utf-8')(stderr) + + return self._run(argv) + except KeyboardInterrupt: + print('mach interrupted by signal or user action. Stopping.') + return 1 + + except Exception as e: + # _run swallows exceptions in invoked handlers and converts them to + # a proper exit code. So, the only scenario where we should get an + # exception here is if _run itself raises. If _run raises, that's a + # bug in mach (or a loaded command module being silly) and thus + # should be reported differently. + self._print_error_header(argv, sys.stdout) + print(MACH_ERROR) + + exc_type, exc_value, exc_tb = sys.exc_info() + stack = traceback.extract_tb(exc_tb) + + self._print_exception(sys.stdout, exc_type, exc_value, stack) + + return 1 + + finally: + sys.stdin = orig_stdin + sys.stdout = orig_stdout + sys.stderr = orig_stderr + + def _run(self, argv): + context = CommandContext(cwd=self.cwd, + settings=self.settings, log_manager=self.log_manager, + commands=Registrar) + + if self.populate_context_handler: + self.populate_context_handler(context) + context = ContextWrapper(context, self.populate_context_handler) + + parser = self.get_argument_parser(context) + + if not len(argv): + # We don't register the usage until here because if it is globally + # registered, argparse always prints it. This is not desired when + # running with --help. + parser.usage = Mach.USAGE + parser.print_usage() + return 0 + + try: + args = parser.parse_args(argv) + except NoCommandError: + print(NO_COMMAND_ERROR) + return 1 + except UnknownCommandError as e: + suggestion_message = SUGGESTED_COMMANDS_MESSAGE % (e.verb, ', '.join(e.suggested_commands)) if e.suggested_commands else '' + print(UNKNOWN_COMMAND_ERROR % (e.verb, e.command, suggestion_message)) + return 1 + except UnrecognizedArgumentError as e: + print(UNRECOGNIZED_ARGUMENT_ERROR % (e.command, + ' '.join(e.arguments))) + return 1 + + # Add JSON logging to a file if requested. + if args.logfile: + self.log_manager.add_json_handler(args.logfile) + + # Up the logging level if requested. + log_level = logging.INFO + if args.verbose: + log_level = logging.DEBUG + + self.log_manager.register_structured_logger(logging.getLogger('mach')) + + write_times = True + if args.log_no_times or 'MACH_NO_WRITE_TIMES' in os.environ: + write_times = False + + # Always enable terminal logging. The log manager figures out if we are + # actually in a TTY or are a pipe and does the right thing. + self.log_manager.add_terminal_logging(level=log_level, + write_interval=args.log_interval, write_times=write_times) + + self.load_settings(args) + + if not hasattr(args, 'mach_handler'): + raise MachError('ArgumentParser result missing mach handler info.') + + handler = getattr(args, 'mach_handler') + cls = handler.cls + + if handler.pass_context: + instance = cls(context) + else: + instance = cls() + + if handler.conditions: + fail_conditions = [] + for c in handler.conditions: + if not c(instance): + fail_conditions.append(c) + + if fail_conditions: + print(self._condition_failed_message(handler.name, fail_conditions)) + return 1 + + fn = getattr(instance, handler.method) + + try: + result = fn(**vars(args.command_args)) + + if not result: + result = 0 + + assert isinstance(result, (int, long)) + + return result + except KeyboardInterrupt as ki: + raise ki + except Exception as e: + exc_type, exc_value, exc_tb = sys.exc_info() + + # The first frame is us and is never used. + stack = traceback.extract_tb(exc_tb)[1:] + + # If we have nothing on the stack, the exception was raised as part + # of calling the @Command method itself. This likely means a + # mismatch between @CommandArgument and arguments to the method. + # e.g. there exists a @CommandArgument without the corresponding + # argument on the method. We handle that here until the module + # loader grows the ability to validate better. + if not len(stack): + print(COMMAND_ERROR) + self._print_exception(sys.stdout, exc_type, exc_value, + traceback.extract_tb(exc_tb)) + return 1 + + # Split the frames into those from the module containing the + # command and everything else. + command_frames = [] + other_frames = [] + + initial_file = stack[0][0] + + for frame in stack: + if frame[0] == initial_file: + command_frames.append(frame) + else: + other_frames.append(frame) + + # If the exception was in the module providing the command, it's + # likely the bug is in the mach command module, not something else. + # If there are other frames, the bug is likely not the mach + # command's fault. + self._print_error_header(argv, sys.stdout) + + if len(other_frames): + print(MODULE_ERROR) + else: + print(COMMAND_ERROR) + + self._print_exception(sys.stdout, exc_type, exc_value, stack) + + return 1 + + def log(self, level, action, params, format_str): + """Helper method to record a structured log event.""" + self.logger.log(level, format_str, + extra={'action': action, 'params': params}) + + @classmethod + def _condition_failed_message(cls, name, conditions): + msg = ['\n'] + for c in conditions: + part = [' %s' % c.__name__] + if c.__doc__ is not None: + part.append(c.__doc__) + msg.append(' - '.join(part)) + return INVALID_COMMAND_CONTEXT % (name, '\n'.join(msg)) + + def _print_error_header(self, argv, fh): + fh.write('Error running mach:\n\n') + fh.write(' ') + fh.write(repr(argv)) + fh.write('\n\n') + + def _print_exception(self, fh, exc_type, exc_value, stack): + fh.write(ERROR_FOOTER) + fh.write('\n') + + for l in traceback.format_exception_only(exc_type, exc_value): + fh.write(l) + + fh.write('\n') + for l in traceback.format_list(stack): + fh.write(l) + + def load_settings(self, args): + """Determine which settings files apply and load them. + + Currently, we only support loading settings from a single file. + Ideally, we support loading from multiple files. This is supported by + the ConfigSettings API. However, that API currently doesn't track where + individual values come from, so if we load from multiple sources then + save, we effectively do a full copy. We don't want this. Until + ConfigSettings does the right thing, we shouldn't expose multi-file + loading. + + We look for a settings file in the following locations. The first one + found wins: + + 1) Command line argument + 2) Environment variable + 3) Default path + """ + # Settings are disabled until integration with command providers is + # worked out. + self.settings = None + return False + + for provider in Registrar.settings_providers: + provider.register_settings() + self.settings.register_provider(provider) + + p = os.path.join(self.cwd, 'mach.ini') + + if args.settings_file: + p = args.settings_file + elif 'MACH_SETTINGS_FILE' in os.environ: + p = os.environ['MACH_SETTINGS_FILE'] + + self.settings.load_file(p) + + return os.path.exists(p) + + def get_argument_parser(self, context): + """Returns an argument parser for the command-line interface.""" + + parser = ArgumentParser(add_help=False, + usage='%(prog)s [global arguments] command [command arguments]') + + # Order is important here as it dictates the order the auto-generated + # help messages are printed. + global_group = parser.add_argument_group('Global Arguments') + + #global_group.add_argument('--settings', dest='settings_file', + # metavar='FILENAME', help='Path to settings file.') + + global_group.add_argument('-v', '--verbose', dest='verbose', + action='store_true', default=False, + help='Print verbose output.') + global_group.add_argument('-l', '--log-file', dest='logfile', + metavar='FILENAME', type=argparse.FileType('ab'), + help='Filename to write log data to.') + global_group.add_argument('--log-interval', dest='log_interval', + action='store_true', default=False, + help='Prefix log line with interval from last message rather ' + 'than relative time. Note that this is NOT execution time ' + 'if there are parallel operations.') + global_group.add_argument('--log-no-times', dest='log_no_times', + action='store_true', default=False, + help='Do not prefix log lines with times. By default, mach will ' + 'prefix each output line with the time since command start.') + global_group.add_argument('-h', '--help', dest='help', + action='store_true', default=False, + help='Show this help message.') + + for args, kwargs in self.global_arguments: + global_group.add_argument(*args, **kwargs) + + # We need to be last because CommandAction swallows all remaining + # arguments and argparse parses arguments in the order they were added. + parser.add_argument('command', action=CommandAction, + registrar=Registrar, context=context) + + return parser diff --git a/python/mach/mach/mixin/__init__.py b/python/mach/mach/mixin/__init__.py new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/python/mach/mach/mixin/__init__.py diff --git a/python/mach/mach/mixin/logging.py b/python/mach/mach/mixin/logging.py new file mode 100644 index 00000000000..5c37b54f1bd --- /dev/null +++ b/python/mach/mach/mixin/logging.py @@ -0,0 +1,55 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, unicode_literals + +import logging + + +class LoggingMixin(object): + """Provides functionality to control logging.""" + + def populate_logger(self, name=None): + """Ensure this class instance has a logger associated with it. + + Users of this mixin that call log() will need to ensure self._logger is + a logging.Logger instance before they call log(). This function ensures + self._logger is defined by populating it if it isn't. + """ + if hasattr(self, '_logger'): + return + + if name is None: + name = '.'.join([self.__module__, self.__class__.__name__]) + + self._logger = logging.getLogger(name) + + def log(self, level, action, params, format_str): + """Log a structured log event. + + A structured log event consists of a logging level, a string action, a + dictionary of attributes, and a formatting string. + + The logging level is one of the logging.* constants, such as + logging.INFO. + + The action string is essentially the enumeration of the event. Each + different type of logged event should have a different action. + + The params dict is the metadata constituting the logged event. + + The formatting string is used to convert the structured message back to + human-readable format. Conversion back to human-readable form is + performed by calling format() on this string, feeding into it the dict + of attributes constituting the event. + + Example Usage + ------------- + + self.log(logging.DEBUG, 'login', {'username': 'johndoe'}, + 'User login: {username}') + """ + self._logger.log(level, format_str, + extra={'action': action, 'params': params}) + diff --git a/python/mach/mach/mixin/process.py b/python/mach/mach/mixin/process.py new file mode 100644 index 00000000000..5b06da3b60e --- /dev/null +++ b/python/mach/mach/mixin/process.py @@ -0,0 +1,175 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This module provides mixins to perform process execution. + +from __future__ import absolute_import, unicode_literals + +import logging +import os +import subprocess +import sys + +from mozprocess.processhandler import ProcessHandlerMixin + +from .logging import LoggingMixin + + +# Perform detection of operating system environment. This is used by command +# execution. We only do this once to save redundancy. Yes, this can fail module +# loading. That is arguably OK. +if 'SHELL' in os.environ: + _current_shell = os.environ['SHELL'] +elif 'MOZILLABUILD' in os.environ: + _current_shell = os.environ['MOZILLABUILD'] + '/msys/bin/sh.exe' +elif 'COMSPEC' in os.environ: + _current_shell = os.environ['COMSPEC'] +else: + raise Exception('Could not detect environment shell!') + +_in_msys = False + +if os.environ.get('MSYSTEM', None) == 'MINGW32': + _in_msys = True + + if not _current_shell.lower().endswith('.exe'): + _current_shell += '.exe' + + +class ProcessExecutionMixin(LoggingMixin): + """Mix-in that provides process execution functionality.""" + + def run_process(self, args=None, cwd=None, append_env=None, + explicit_env=None, log_name=None, log_level=logging.INFO, + line_handler=None, require_unix_environment=False, + ensure_exit_code=0, ignore_children=False, pass_thru=False): + """Runs a single process to completion. + + Takes a list of arguments to run where the first item is the + executable. Runs the command in the specified directory and + with optional environment variables. + + append_env -- Dict of environment variables to append to the current + set of environment variables. + explicit_env -- Dict of environment variables to set for the new + process. Any existing environment variables will be ignored. + + require_unix_environment if True will ensure the command is executed + within a UNIX environment. Basically, if we are on Windows, it will + execute the command via an appropriate UNIX-like shell. + + ignore_children is proxied to mozprocess's ignore_children. + + ensure_exit_code is used to ensure the exit code of a process matches + what is expected. If it is an integer, we raise an Exception if the + exit code does not match this value. If it is True, we ensure the exit + code is 0. If it is False, we don't perform any exit code validation. + + pass_thru is a special execution mode where the child process inherits + this process's standard file handles (stdin, stdout, stderr) as well as + additional file descriptors. It should be used for interactive processes + where buffering from mozprocess could be an issue. pass_thru does not + use mozprocess. Therefore, arguments like log_name, line_handler, + and ignore_children have no effect. + """ + args = self._normalize_command(args, require_unix_environment) + + self.log(logging.INFO, 'new_process', {'args': args}, ' '.join(args)) + + def handleLine(line): + # Converts str to unicode on Python 2 and bytes to str on Python 3. + if isinstance(line, bytes): + line = line.decode(sys.stdout.encoding or 'utf-8', 'replace') + + if line_handler: + line_handler(line) + + if not log_name: + return + + self.log(log_level, log_name, {'line': line.rstrip()}, '{line}') + + use_env = {} + if explicit_env: + use_env = explicit_env + else: + use_env.update(os.environ) + + if append_env: + use_env.update(append_env) + + self.log(logging.DEBUG, 'process', {'env': use_env}, 'Environment: {env}') + + # There is a bug in subprocess where it doesn't like unicode types in + # environment variables. Here, ensure all unicode are converted to + # binary. utf-8 is our globally assumed default. If the caller doesn't + # want UTF-8, they shouldn't pass in a unicode instance. + normalized_env = {} + for k, v in use_env.items(): + if isinstance(k, unicode): + k = k.encode('utf-8', 'strict') + + if isinstance(v, unicode): + v = v.encode('utf-8', 'strict') + + normalized_env[k] = v + + use_env = normalized_env + + if pass_thru: + proc = subprocess.Popen(args, cwd=cwd, env=use_env) + status = None + # Leave it to the subprocess to handle Ctrl+C. If it terminates as + # a result of Ctrl+C, proc.wait() will return a status code, and, + # we get out of the loop. If it doesn't, like e.g. gdb, we continue + # waiting. + while status is None: + try: + status = proc.wait() + except KeyboardInterrupt: + pass + else: + p = ProcessHandlerMixin(args, cwd=cwd, env=use_env, + processOutputLine=[handleLine], universal_newlines=True, + ignore_children=ignore_children) + p.run() + p.processOutput() + status = p.wait() + + if ensure_exit_code is False: + return status + + if ensure_exit_code is True: + ensure_exit_code = 0 + + if status != ensure_exit_code: + raise Exception('Process executed with non-0 exit code: %s' % args) + + return status + + def _normalize_command(self, args, require_unix_environment): + """Adjust command arguments to run in the necessary environment. + + This exists mainly to facilitate execution of programs requiring a *NIX + shell when running on Windows. The caller specifies whether a shell + environment is required. If it is and we are running on Windows but + aren't running in the UNIX-like msys environment, then we rewrite the + command to execute via a shell. + """ + assert isinstance(args, list) and len(args) + + if not require_unix_environment or not _in_msys: + return args + + # Always munge Windows-style into Unix style for the command. + prog = args[0].replace('\\', '/') + + # PyMake removes the C: prefix. But, things seem to work here + # without it. Not sure what that's about. + + # We run everything through the msys shell. We need to use + # '-c' and pass all the arguments as one argument because that is + # how sh works. + cline = subprocess.list2cmdline([prog] + args[1:]) + return [_current_shell, '-c', cline] diff --git a/python/mach/mach/registrar.py b/python/mach/mach/registrar.py new file mode 100644 index 00000000000..a2de24b2d04 --- /dev/null +++ b/python/mach/mach/registrar.py @@ -0,0 +1,65 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +from .base import MachError + + +class MachRegistrar(object): + """Container for mach command and config providers.""" + + def __init__(self): + self.command_handlers = {} + self.commands_by_category = {} + self.settings_providers = set() + self.categories = {} + self.require_conditions = False + + def register_command_handler(self, handler): + name = handler.name + + if not handler.category: + raise MachError('Cannot register a mach command without a ' + 'category: %s' % name) + + if handler.category not in self.categories: + raise MachError('Cannot register a command to an undefined ' + 'category: %s -> %s' % (name, handler.category)) + + self.command_handlers[name] = handler + self.commands_by_category[handler.category].add(name) + + def register_settings_provider(self, cls): + self.settings_providers.add(cls) + + def register_category(self, name, title, description, priority=50): + self.categories[name] = (title, description, priority) + self.commands_by_category[name] = set() + + def dispatch(self, name, context=None, **args): + """Dispatch/run a command. + + Commands can use this to call other commands. + """ + + # TODO The logic in this function overlaps with code in + # mach.main.Main._run() and should be consolidated. + handler = self.command_handlers[name] + cls = handler.cls + + if handler.pass_context and not context: + raise Exception('mach command class requires context.') + + if handler.pass_context: + instance = cls(context) + else: + instance = cls() + + fn = getattr(instance, handler.method) + + return fn(**args) or 0 + + +Registrar = MachRegistrar() diff --git a/python/mach/mach/terminal.py b/python/mach/mach/terminal.py new file mode 100644 index 00000000000..cdc3966575e --- /dev/null +++ b/python/mach/mach/terminal.py @@ -0,0 +1,75 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +"""This file contains code for interacting with terminals. + +All the terminal interaction code is consolidated so the complexity can be in +one place, away from code that is commonly looked at. +""" + +from __future__ import print_function, unicode_literals + +import logging +import sys + + +class LoggingHandler(logging.Handler): + """Custom logging handler that works with terminal window dressing. + + This is alternative terminal logging handler which contains smarts for + emitting terminal control characters properly. Currently, it has generic + support for "footer" elements at the bottom of the screen. Functionality + can be added when needed. + """ + def __init__(self): + logging.Handler.__init__(self) + + self.fh = sys.stdout + self.footer = None + + def flush(self): + self.acquire() + + try: + self.fh.flush() + finally: + self.release() + + def emit(self, record): + msg = self.format(record) + + if self.footer: + self.footer.clear() + + self.fh.write(msg) + self.fh.write('\n') + + if self.footer: + self.footer.draw() + + # If we don't flush, the footer may not get drawn. + self.flush() + + +class TerminalFooter(object): + """Represents something drawn on the bottom of a terminal.""" + def __init__(self, terminal): + self.t = terminal + self.fh = sys.stdout + + def _clear_lines(self, n): + for i in xrange(n): + self.fh.write(self.t.move_x(0)) + self.fh.write(self.t.clear_eol()) + self.fh.write(self.t.move_up()) + + self.fh.write(self.t.move_down()) + self.fh.write(self.t.move_x(0)) + + def clear(self): + raise Exception('clear() must be implemented.') + + def draw(self): + raise Exception('draw() must be implemented.') + diff --git a/python/mach/mach/test/__init__.py b/python/mach/mach/test/__init__.py new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/python/mach/mach/test/__init__.py diff --git a/python/mach/mach/test/common.py b/python/mach/mach/test/common.py new file mode 100644 index 00000000000..1c4b1ea90ac --- /dev/null +++ b/python/mach/mach/test/common.py @@ -0,0 +1,40 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +from StringIO import StringIO +import os +import unittest + +from mach.main import Mach +from mach.base import CommandContext + +here = os.path.abspath(os.path.dirname(__file__)) + +class TestBase(unittest.TestCase): + provider_dir = os.path.join(here, 'providers') + + def _run_mach(self, args, provider_file=None, entry_point=None, context_handler=None): + m = Mach(os.getcwd()) + m.define_category('testing', 'Mach unittest', 'Testing for mach core', 10) + m.populate_context_handler = context_handler + + if provider_file: + m.load_commands_from_file(os.path.join(self.provider_dir, provider_file)) + + if entry_point: + m.load_commands_from_entry_point(entry_point) + + stdout = StringIO() + stderr = StringIO() + stdout.encoding = 'UTF-8' + stderr.encoding = 'UTF-8' + + try: + result = m.run(args, stdout=stdout, stderr=stderr) + except SystemExit: + result = None + + return (result, stdout.getvalue(), stderr.getvalue()) diff --git a/python/mach/mach/test/providers/__init__.py b/python/mach/mach/test/providers/__init__.py new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/python/mach/mach/test/providers/__init__.py diff --git a/python/mach/mach/test/providers/basic.py b/python/mach/mach/test/providers/basic.py new file mode 100644 index 00000000000..d10856289b1 --- /dev/null +++ b/python/mach/mach/test/providers/basic.py @@ -0,0 +1,15 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import unicode_literals + +from mach.decorators import ( + CommandProvider, + Command, +) + +@CommandProvider +class ConditionsProvider(object): + @Command('cmd_foo', category='testing') + def run_foo(self): + pass diff --git a/python/mach/mach/test/providers/conditions.py b/python/mach/mach/test/providers/conditions.py new file mode 100644 index 00000000000..a95429752d4 --- /dev/null +++ b/python/mach/mach/test/providers/conditions.py @@ -0,0 +1,53 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +from mach.decorators import ( + CommandProvider, + Command, +) + +def is_foo(cls): + """Foo must be true""" + return cls.foo + +def is_bar(cls): + """Bar must be true""" + return cls.bar + +@CommandProvider +class ConditionsProvider(object): + foo = True + bar = False + + @Command('cmd_foo', category='testing', conditions=[is_foo]) + def run_foo(self): + pass + + @Command('cmd_bar', category='testing', conditions=[is_bar]) + def run_bar(self): + pass + + @Command('cmd_foobar', category='testing', conditions=[is_foo, is_bar]) + def run_foobar(self): + pass + +@CommandProvider +class ConditionsContextProvider(object): + def __init__(self, context): + self.foo = context.foo + self.bar = context.bar + + @Command('cmd_foo_ctx', category='testing', conditions=[is_foo]) + def run_foo(self): + pass + + @Command('cmd_bar_ctx', category='testing', conditions=[is_bar]) + def run_bar(self): + pass + + @Command('cmd_foobar_ctx', category='testing', conditions=[is_foo, is_bar]) + def run_foobar(self): + pass diff --git a/python/mach/mach/test/providers/conditions_invalid.py b/python/mach/mach/test/providers/conditions_invalid.py new file mode 100644 index 00000000000..22284d4dcad --- /dev/null +++ b/python/mach/mach/test/providers/conditions_invalid.py @@ -0,0 +1,16 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +from mach.decorators import ( + CommandProvider, + Command, +) + +@CommandProvider +class ConditionsProvider(object): + @Command('cmd_foo', category='testing', conditions=["invalid"]) + def run_foo(self): + pass diff --git a/python/mach/mach/test/providers/throw.py b/python/mach/mach/test/providers/throw.py new file mode 100644 index 00000000000..06bee01eec7 --- /dev/null +++ b/python/mach/mach/test/providers/throw.py @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import time + +from mach.decorators import ( + CommandArgument, + CommandProvider, + Command, +) + +from mach.test.providers import throw2 + + +@CommandProvider +class TestCommandProvider(object): + @Command('throw', category='testing') + @CommandArgument('--message', '-m', default='General Error') + def throw(self, message): + raise Exception(message) + + @Command('throw_deep', category='testing') + @CommandArgument('--message', '-m', default='General Error') + def throw_deep(self, message): + throw2.throw_deep(message) + diff --git a/python/mach/mach/test/providers/throw2.py b/python/mach/mach/test/providers/throw2.py new file mode 100644 index 00000000000..af0a23fcfe7 --- /dev/null +++ b/python/mach/mach/test/providers/throw2.py @@ -0,0 +1,13 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This file exists to trigger the differences in mach error reporting between +# exceptions that occur in mach command modules themselves and in the things +# they call. + +def throw_deep(message): + return throw_real(message) + +def throw_real(message): + raise Exception(message) diff --git a/python/mach/mach/test/test_conditions.py b/python/mach/mach/test/test_conditions.py new file mode 100644 index 00000000000..532a8316062 --- /dev/null +++ b/python/mach/mach/test/test_conditions.py @@ -0,0 +1,82 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import os + +from mach.base import MachError +from mach.main import Mach +from mach.test.common import TestBase + +from mozunit import main + + +def _populate_context(context, key=None): + if key is None: + return + if key == 'foo': + return True + if key == 'bar': + return False + raise AttributeError(key) + +class TestConditions(TestBase): + """Tests for conditionally filtering commands.""" + + def _run_mach(self, args, context_handler=None): + return TestBase._run_mach(self, args, 'conditions.py', + context_handler=context_handler) + + + def test_conditions_pass(self): + """Test that a command which passes its conditions is runnable.""" + + self.assertEquals((0, '', ''), self._run_mach(['cmd_foo'])) + self.assertEquals((0, '', ''), self._run_mach(['cmd_foo_ctx'], _populate_context)) + + def test_invalid_context_message(self): + """Test that commands which do not pass all their conditions + print the proper failure message.""" + + def is_bar(): + """Bar must be true""" + fail_conditions = [is_bar] + + for name in ('cmd_bar', 'cmd_foobar'): + result, stdout, stderr = self._run_mach([name]) + self.assertEquals(1, result) + + fail_msg = Mach._condition_failed_message(name, fail_conditions) + self.assertEquals(fail_msg.rstrip(), stdout.rstrip()) + + for name in ('cmd_bar_ctx', 'cmd_foobar_ctx'): + result, stdout, stderr = self._run_mach([name], _populate_context) + self.assertEquals(1, result) + + fail_msg = Mach._condition_failed_message(name, fail_conditions) + self.assertEquals(fail_msg.rstrip(), stdout.rstrip()) + + def test_invalid_type(self): + """Test that a condition which is not callable raises an exception.""" + + m = Mach(os.getcwd()) + m.define_category('testing', 'Mach unittest', 'Testing for mach core', 10) + self.assertRaises(MachError, m.load_commands_from_file, + os.path.join(self.provider_dir, 'conditions_invalid.py')) + + def test_help_message(self): + """Test that commands that are not runnable do not show up in help.""" + + result, stdout, stderr = self._run_mach(['help'], _populate_context) + self.assertIn('cmd_foo', stdout) + self.assertNotIn('cmd_bar', stdout) + self.assertNotIn('cmd_foobar', stdout) + self.assertIn('cmd_foo_ctx', stdout) + self.assertNotIn('cmd_bar_ctx', stdout) + self.assertNotIn('cmd_foobar_ctx', stdout) + + +if __name__ == '__main__': + main() diff --git a/python/mach/mach/test/test_config.py b/python/mach/mach/test/test_config.py new file mode 100644 index 00000000000..cebd47a7d6c --- /dev/null +++ b/python/mach/mach/test/test_config.py @@ -0,0 +1,264 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import unicode_literals + +import sys +import unittest + +from mozfile.mozfile import NamedTemporaryFile + +from mach.config import ( + AbsolutePathType, + BooleanType, + ConfigProvider, + ConfigSettings, + IntegerType, + PathType, + PositiveIntegerType, + RelativePathType, + StringType, +) + +from mozunit import main + + +if sys.version_info[0] == 3: + str_type = str +else: + str_type = basestring + +CONFIG1 = r""" +[foo] + +bar = bar_value +baz = /baz/foo.c +""" + +CONFIG2 = r""" +[foo] + +bar = value2 +""" + +class Provider1(ConfigProvider): + @classmethod + def _register_settings(cls): + cls.register_setting('foo', 'bar', StringType) + cls.register_setting('foo', 'baz', AbsolutePathType) + +Provider1.register_settings() + +class ProviderDuplicate(ConfigProvider): + @classmethod + def _register_settings(cls): + cls.register_setting('dupesect', 'foo', StringType) + cls.register_setting('dupesect', 'foo', StringType) + +class TestConfigProvider(unittest.TestCase): + def test_construct(self): + s = Provider1.config_settings + + self.assertEqual(len(s), 1) + self.assertIn('foo', s) + + self.assertEqual(len(s['foo']), 2) + self.assertIn('bar', s['foo']) + self.assertIn('baz', s['foo']) + + def test_duplicate_option(self): + with self.assertRaises(Exception): + ProviderDuplicate.register_settings() + + +class Provider2(ConfigProvider): + @classmethod + def _register_settings(cls): + cls.register_setting('a', 'string', StringType) + cls.register_setting('a', 'boolean', BooleanType) + cls.register_setting('a', 'pos_int', PositiveIntegerType) + cls.register_setting('a', 'int', IntegerType) + cls.register_setting('a', 'abs_path', AbsolutePathType) + cls.register_setting('a', 'rel_path', RelativePathType) + cls.register_setting('a', 'path', PathType) + +Provider2.register_settings() + +class TestConfigSettings(unittest.TestCase): + def test_empty(self): + s = ConfigSettings() + + self.assertEqual(len(s), 0) + self.assertNotIn('foo', s) + + def test_simple(self): + s = ConfigSettings() + s.register_provider(Provider1) + + self.assertEqual(len(s), 1) + self.assertIn('foo', s) + + foo = s['foo'] + foo = s.foo + + self.assertEqual(len(foo), 2) + + self.assertIn('bar', foo) + self.assertIn('baz', foo) + + foo['bar'] = 'value1' + self.assertEqual(foo['bar'], 'value1') + self.assertEqual(foo['bar'], 'value1') + + def test_assignment_validation(self): + s = ConfigSettings() + s.register_provider(Provider2) + + a = s.a + + # Assigning an undeclared setting raises. + with self.assertRaises(KeyError): + a.undefined = True + + with self.assertRaises(KeyError): + a['undefined'] = True + + # Basic type validation. + a.string = 'foo' + a.string = 'foo' + + with self.assertRaises(TypeError): + a.string = False + + a.boolean = True + a.boolean = False + + with self.assertRaises(TypeError): + a.boolean = 'foo' + + a.pos_int = 5 + a.pos_int = 0 + + with self.assertRaises(ValueError): + a.pos_int = -1 + + with self.assertRaises(TypeError): + a.pos_int = 'foo' + + a.int = 5 + a.int = 0 + a.int = -5 + + with self.assertRaises(TypeError): + a.int = 1.24 + + with self.assertRaises(TypeError): + a.int = 'foo' + + a.abs_path = '/home/gps' + + with self.assertRaises(ValueError): + a.abs_path = 'home/gps' + + a.rel_path = 'home/gps' + a.rel_path = './foo/bar' + a.rel_path = 'foo.c' + + with self.assertRaises(ValueError): + a.rel_path = '/foo/bar' + + a.path = '/home/gps' + a.path = 'foo.c' + a.path = 'foo/bar' + a.path = './foo' + + def test_retrieval_type(self): + s = ConfigSettings() + s.register_provider(Provider2) + + a = s.a + + a.string = 'foo' + a.boolean = True + a.pos_int = 12 + a.int = -4 + a.abs_path = '/home/gps' + a.rel_path = 'foo.c' + a.path = './foo/bar' + + self.assertIsInstance(a.string, str_type) + self.assertIsInstance(a.boolean, bool) + self.assertIsInstance(a.pos_int, int) + self.assertIsInstance(a.int, int) + self.assertIsInstance(a.abs_path, str_type) + self.assertIsInstance(a.rel_path, str_type) + self.assertIsInstance(a.path, str_type) + + def test_file_reading_single(self): + temp = NamedTemporaryFile(mode='wt') + temp.write(CONFIG1) + temp.flush() + + s = ConfigSettings() + s.register_provider(Provider1) + + s.load_file(temp.name) + + self.assertEqual(s.foo.bar, 'bar_value') + + def test_file_reading_multiple(self): + """Loading multiple files has proper overwrite behavior.""" + temp1 = NamedTemporaryFile(mode='wt') + temp1.write(CONFIG1) + temp1.flush() + + temp2 = NamedTemporaryFile(mode='wt') + temp2.write(CONFIG2) + temp2.flush() + + s = ConfigSettings() + s.register_provider(Provider1) + + s.load_files([temp1.name, temp2.name]) + + self.assertEqual(s.foo.bar, 'value2') + + def test_file_reading_missing(self): + """Missing files should silently be ignored.""" + + s = ConfigSettings() + + s.load_file('/tmp/foo.ini') + + def test_file_writing(self): + s = ConfigSettings() + s.register_provider(Provider2) + + s.a.string = 'foo' + s.a.boolean = False + + temp = NamedTemporaryFile('wt') + s.write(temp) + temp.flush() + + s2 = ConfigSettings() + s2.register_provider(Provider2) + + s2.load_file(temp.name) + + self.assertEqual(s.a.string, s2.a.string) + self.assertEqual(s.a.boolean, s2.a.boolean) + + def test_write_pot(self): + s = ConfigSettings() + s.register_provider(Provider1) + s.register_provider(Provider2) + + # Just a basic sanity test. + temp = NamedTemporaryFile('wt') + s.write_pot(temp) + temp.flush() + + +if __name__ == '__main__': + main() diff --git a/python/mach/mach/test/test_entry_point.py b/python/mach/mach/test/test_entry_point.py new file mode 100644 index 00000000000..5bd2c279d45 --- /dev/null +++ b/python/mach/mach/test/test_entry_point.py @@ -0,0 +1,60 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import unicode_literals + +import imp +import os +import sys + +from mach.base import MachError +from mach.test.common import TestBase +from mock import patch + +from mozunit import main + + +here = os.path.abspath(os.path.dirname(__file__)) + +class Entry(): + """Stub replacement for pkg_resources.EntryPoint""" + def __init__(self, providers): + self.providers = providers + + def load(self): + def _providers(): + return self.providers + return _providers + +class TestEntryPoints(TestBase): + """Test integrating with setuptools entry points""" + provider_dir = os.path.join(here, 'providers') + + def _run_mach(self): + return TestBase._run_mach(self, ['help'], entry_point='mach.providers') + + @patch('pkg_resources.iter_entry_points') + def test_load_entry_point_from_directory(self, mock): + # Ensure parent module is present otherwise we'll (likely) get + # an error due to unknown parent. + if b'mach.commands' not in sys.modules: + mod = imp.new_module(b'mach.commands') + sys.modules[b'mach.commands'] = mod + + mock.return_value = [Entry(['providers'])] + # Mach error raised due to conditions_invalid.py + with self.assertRaises(MachError): + self._run_mach() + + @patch('pkg_resources.iter_entry_points') + def test_load_entry_point_from_file(self, mock): + mock.return_value = [Entry([os.path.join('providers', 'basic.py')])] + + result, stdout, stderr = self._run_mach() + self.assertIsNone(result) + self.assertIn('cmd_foo', stdout) + + +# Not enabled in automation because tests are failing. +#if __name__ == '__main__': +# main() diff --git a/python/mach/mach/test/test_error_output.py b/python/mach/mach/test/test_error_output.py new file mode 100644 index 00000000000..25553f96bc4 --- /dev/null +++ b/python/mach/mach/test/test_error_output.py @@ -0,0 +1,39 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +from mach.main import ( + COMMAND_ERROR, + MODULE_ERROR +) +from mach.test.common import TestBase + +from mozunit import main + + +class TestErrorOutput(TestBase): + + def _run_mach(self, args): + return TestBase._run_mach(self, args, 'throw.py') + + def test_command_error(self): + result, stdout, stderr = self._run_mach(['throw', '--message', + 'Command Error']) + + self.assertEqual(result, 1) + + self.assertIn(COMMAND_ERROR, stdout) + + def test_invoked_error(self): + result, stdout, stderr = self._run_mach(['throw_deep', '--message', + 'Deep stack']) + + self.assertEqual(result, 1) + + self.assertIn(MODULE_ERROR, stdout) + + +if __name__ == '__main__': + main() diff --git a/python/mach/mach/test/test_logger.py b/python/mach/mach/test/test_logger.py new file mode 100644 index 00000000000..05592845e7f --- /dev/null +++ b/python/mach/mach/test/test_logger.py @@ -0,0 +1,47 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, unicode_literals + +import logging +import time +import unittest + +from mach.logging import StructuredHumanFormatter + +from mozunit import main + + +class DummyLogger(logging.Logger): + def __init__(self, cb): + logging.Logger.__init__(self, 'test') + + self._cb = cb + + def handle(self, record): + self._cb(record) + + +class TestStructuredHumanFormatter(unittest.TestCase): + def test_non_ascii_logging(self): + # Ensures the formatter doesn't choke when non-ASCII characters are + # present in printed parameters. + formatter = StructuredHumanFormatter(time.time()) + + def on_record(record): + result = formatter.format(record) + relevant = result[9:] + + self.assertEqual(relevant, 'Test: s\xe9curit\xe9') + + logger = DummyLogger(on_record) + + value = 's\xe9curit\xe9' + + logger.log(logging.INFO, 'Test: {utf}', + extra={'action': 'action', 'params': {'utf': value}}) + + +if __name__ == '__main__': + main() diff --git a/python/mach/setup.py b/python/mach/setup.py new file mode 100644 index 00000000000..511a6a32277 --- /dev/null +++ b/python/mach/setup.py @@ -0,0 +1,38 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +try: + from setuptools import setup +except: + from distutils.core import setup + + +VERSION = '0.3' + +README = open('README.rst').read() + +setup( + name='mach', + description='Generic command line command dispatching framework.', + long_description=README, + license='MPL 2.0', + author='Gregory Szorc', + author_email='gregory.szorc@gmail.com', + url='https://developer.mozilla.org/en-US/docs/Developer_Guide/mach', + packages=['mach'], + version=VERSION, + classifiers=[ + 'Environment :: Console', + 'Development Status :: 3 - Alpha', + 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)', + 'Natural Language :: English', + ], + install_requires=[ + 'blessings', + 'mozfile', + 'mozprocess', + ], + tests_require=['mock'], +) + |