Source code for rally.verification.manager

# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import abc
import inspect
import io
import os
import re
import shutil
import sys

import pkg_resources

from rally.common.io import subunit_v2
from rally.common import logging
from rally.common.plugin import plugin
from rally import exceptions
from rally.verification import context
from rally.verification import utils


LOG = logging.getLogger(__name__)

URL_RE = re.compile(
    r"^(?:(?:http|ftp)s?|ssh)://"  # http:// or https://
    r"(?:(?:[A-Z0-9](?:[[email protected]]{0,61}[A-Z0-9])?\.)+"  # domain
    r"(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|"  # domain
    r"localhost|"  # localhost
    r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})"  # IP
    r"(?::\d+)?"  # optional port
    r"(?:/?|[/?]\S+)$", re.IGNORECASE)


class VerifierSetupFailure(exceptions.RallyException):
    error_code = 224
    msg_fmt = "Failed to set up verifier '%(verifier)s': %(message)s"


def configure(name, platform="default", default_repo=None,
              default_version=None, context=None):
    """Decorator to configure plugin's attributes.

    :param name: Plugin name that is used for searching purpose
    :param platform: Plugin platform
    :param default_repo: Default repository to clone
    :param default_version: Default version to checkout
    :param context: List of contexts that should be executed for verification
    """
    def decorator(plugin_inst):
        plugin_inst = plugin.configure(name, platform=platform)(plugin_inst)
        plugin_inst._meta_set("default_repo", default_repo)
        plugin_inst._meta_set("default_version", default_version)
        plugin_inst._meta_set("context", context or {})
        return plugin_inst

    return decorator


[docs]@plugin.base() class VerifierManager(plugin.Plugin, metaclass=abc.ABCMeta): """Verifier base class. This class provides an interface for operating specific tool. """ # These dicts will be used for building docs. PS: we should find a better # place for them RUN_ARGS = {"pattern": "a regular expression of tests to launch.", "concurrency": "Number of processes to be used for launching " "tests. In case of 0 value, number of processes" " will be equal to number of CPU cores.", "load_list": "a list of tests to launch.", "skip_list": "a list of tests to skip (actually, it is a dict " "where keys are names of tests, values are " "reasons).", "xfail_list": "a list of tests that are expected to fail " "(actually, it is a dict where keys are names " "of tests, values are reasons)."} @classmethod def _get_doc(cls): run_args = {} for parent in inspect.getmro(cls): if hasattr(parent, "RUN_ARGS"): for k, v in parent.RUN_ARGS.items(): run_args.setdefault(k, v) doc = cls.__doc__ or "" doc += "\n**Running arguments**:\n\n%s" % "\n".join( sorted(["* *%s*: %s" % (k, v) for k, v in run_args.items()])) doc += "\n\n**Installation arguments**:\n\n" doc += ("* *system_wide*: Whether or not to use the system-wide " "environment for verifier instead of a virtual environment. " "Defaults to False.\n" "* *source*: Path or URL to the repo to clone verifier from." " Defaults to %(default_source)s\n" "* *version*: Branch, tag or commit ID to checkout before " "verifier installation. Defaults to '%(default_version)s'.\n" % {"default_source": cls._meta_get("default_repo"), "default_version": cls._meta_get( "default_version") or "master"}) return doc def __init__(self, verifier): """Init a verifier manager. :param verifier: `rally.common.objects.verifier.Verifier` instance """ self.verifier = verifier @property def base_dir(self): return os.path.expanduser( "~/.rally/verification/verifier-%s" % self.verifier.uuid) @property def home_dir(self): return os.path.join(self.base_dir, "for-deployment-%s" % self.verifier.deployment["uuid"]) @property def repo_dir(self): return os.path.join(self.base_dir, "repo") @property def venv_dir(self): return os.path.join(self.base_dir, ".venv") @property def environ(self): env = os.environ.copy() if not self.verifier.system_wide: # activate virtual environment env["VIRTUAL_ENV"] = self.venv_dir env["PATH"] = "%s:%s" % ( os.path.join(self.venv_dir, "bin"), env["PATH"]) return env
[docs] def validate_args(self, args): """Validate given arguments to be used for running verification. :param args: A dict of arguments with values """ # NOTE(andreykurilin): By default we do not use jsonschema here. # So it cannot be extended by inheritors => requires duplication. if "pattern" in args: if not isinstance(args["pattern"], str): raise exceptions.ValidationError( "'pattern' argument should be a string.") if "concurrency" in args: if (not isinstance(args["concurrency"], int) or args["concurrency"] < 0): raise exceptions.ValidationError( "'concurrency' argument should be a positive integer or " "zero.") if "load_list" in args: if not isinstance(args["load_list"], list): raise exceptions.ValidationError( "'load_list' argument should be a list of tests.") if "skip_list" in args: if not isinstance(args["skip_list"], dict): raise exceptions.ValidationError( "'skip_list' argument should be a dict of tests " "where keys are test names and values are reasons.") if "xfail_list" in args: if not isinstance(args["xfail_list"], dict): raise exceptions.ValidationError( "'xfail_list' argument should be a dict of tests " "where keys are test names and values are reasons.")
def validate(self, run_args): """Validate a verifier context and run arguments.""" context.ContextManager.validate(self._meta_get("context")) self.validate_args(run_args) def _clone(self): """Clone a repo and switch to a certain version.""" source = self.verifier.source or self._meta_get("default_repo") if not source or ( not URL_RE.match(source) and not os.path.exists(source)): raise exceptions.RallyException("Source path '%s' is not valid." % source) if logging.is_debug(): LOG.debug("Cloning verifier repo from %s into %s." % (source, self.repo_dir)) else: LOG.info("Cloning verifier repo from %s." % source) cmd = ["git", "clone", source, self.repo_dir] default_version = self._meta_get("default_version") if default_version and default_version != "master": cmd.extend(["-b", default_version]) utils.check_output(cmd) version = self.verifier.version if version: LOG.info("Switching verifier repo to the '%s' version." % version) utils.check_output(["git", "checkout", version], cwd=self.repo_dir) else: output = utils.check_output(["git", "describe", "--all"], cwd=self.repo_dir).strip() if output.startswith("heads/"): # it is a branch version = output[6:] else: head = utils.check_output(["git", "rev-parse", "HEAD"], cwd=self.repo_dir).strip() if output.endswith(head[:7]): # it is a commit ID version = head else: # it is a tag version = output self.verifier.update_properties(version=version)
[docs] def install(self): """Clone and install a verifier.""" utils.create_dir(self.base_dir) self._clone() if self.verifier.system_wide: self.check_system_wide() else: self.install_venv()
[docs] def uninstall(self, full=False): """Uninstall a verifier. :param full: If False (default behaviour), only deployment-specific data will be removed """ path = self.base_dir if full else self.home_dir if os.path.exists(path): shutil.rmtree(path)
def install_venv(self): """Install a virtual environment for a verifier.""" if os.path.exists(self.venv_dir): # NOTE(andreykurilin): It is necessary to remove the old env while # performing update action. LOG.info("Deleting old virtual environment.") shutil.rmtree(self.venv_dir) LOG.info("Creating virtual environment. It may take a few minutes.") LOG.debug("Initializing virtual environment in %s directory." % self.venv_dir) utils.check_output(["virtualenv", "-p", sys.executable, self.venv_dir], cwd=self.repo_dir, msg_on_err="Failed to initialize virtual env " "in %s directory." % self.venv_dir) LOG.debug("Installing verifier in virtual environment.") # NOTE(ylobankov): Use 'develop mode' installation to provide an # ability to advanced users to change tests or # develop new ones in verifier repo on the fly. utils.check_output(["pip", "install", "-e", "./"], cwd=self.repo_dir, env=self.environ) def check_system_wide(self, reqs_file_path=None): """Check that all required verifier packages are installed.""" LOG.debug("Checking system-wide packages for verifier.") reqs_file_path = reqs_file_path or os.path.join(self.repo_dir, "requirements.txt") with open(reqs_file_path) as f: required_packages = [ p for p in f.read().split("\n") if p.strip() and not p.startswith("#") ] try: pkg_resources.require(required_packages) except (pkg_resources.DistributionNotFound, pkg_resources.VersionConflict) as e: raise VerifierSetupFailure(e.report(), verifier=self.verifier.name) def checkout(self, version): """Switch a verifier repo.""" LOG.info("Switching verifier repo to the '%s' version." % version) utils.check_output(["git", "checkout", "master"], cwd=self.repo_dir) utils.check_output(["git", "remote", "update"], cwd=self.repo_dir) utils.check_output(["git", "pull"], cwd=self.repo_dir) utils.check_output(["git", "checkout", version], cwd=self.repo_dir)
[docs] def configure(self, extra_options=None): """Configure a verifier. :param extra_options: a dictionary with external verifier specific options for configuration. :raises NotImplementedError: This feature is verifier-specific, so you should override this method in your plugin if it supports configuration """ raise NotImplementedError( "'%s' verifiers don't support configuration at all." % self.get_name())
[docs] def is_configured(self): """Check whether a verifier is configured or not.""" return True
[docs] def get_configuration(self): """Get verifier configuration (e.g., the config file content).""" return ""
[docs] def override_configuration(self, new_configuration): """Override verifier configuration. :param new_configuration: Content which should be used while overriding existing configuration :raises NotImplementedError: This feature is verifier-specific, so you should override this method in your plugin if it supports configuration """ raise NotImplementedError( "'%s' verifiers don't support configuration at all." % self.get_name())
[docs] def extend_configuration(self, extra_options): """Extend verifier configuration with new options. :param extra_options: Options to be used for extending configuration :raises NotImplementedError: This feature is verifier-specific, so you should override this method in your plugin if it supports configuration """ raise NotImplementedError( "'%s' verifiers don't support configuration at all." % self.get_name())
[docs] def install_extension(self, source, version=None, extra_settings=None): """Install a verifier extension. :param source: Path or URL to the repo to clone verifier extension from :param version: Branch, tag or commit ID to checkout before verifier extension installation :param extra_settings: Extra installation settings for verifier extension :raises NotImplementedError: This feature is verifier-specific, so you should override this method in your plugin if it supports extensions """ raise NotImplementedError( "'%s' verifiers don't support extensions." % self.get_name())
[docs] def list_extensions(self): """List all verifier extensions. Every extension is a dict object which contains name and entry_point keys. example: { "name": p.name, "entry_point": p.entry_point_target } """ return []
[docs] def uninstall_extension(self, name): """Uninstall a verifier extension. :param name: Name of extension to uninstall :raises NotImplementedError: This feature is verifier-specific, so you should override this method in your plugin if it supports extensions """ raise NotImplementedError( "'%s' verifiers don't support extensions." % self.get_name())
[docs] @abc.abstractmethod def list_tests(self, pattern=""): """List all verifier tests. :param pattern: Filter tests by given pattern """
def parse_results(self, results_data): """Parse subunit results data of a test run.""" # TODO(andreykurilin): Support more formats. return subunit_v2.parse(io.StringIO(results_data))
[docs] @abc.abstractmethod def run(self, context): """Run verifier tests. Verification Component API expects that this method should return an object. There is no special class, you do it as you want, but it should have the following properties: .. code-block:: none <object>.totals = { "tests_count": <total tests count>, "tests_duration": <total tests duration>, "failures": <total count of failed tests>, "skipped": <total count of skipped tests>, "success": <total count of successful tests>, "unexpected_success": <total count of unexpected successful tests>, "expected_failures": <total count of expected failed tests> } <object>.tests = { <test_id>: { "status": <test status>, "name": <test name>, "duration": <test duration>, "reason": <reason>, # optional "traceback": <traceback> # optional }, ... } """