diff options
Diffstat (limited to 'dep/dep.py')
| -rw-r--r-- | dep/dep.py | 488 |
1 files changed, 488 insertions, 0 deletions
diff --git a/dep/dep.py b/dep/dep.py new file mode 100644 index 00000000..6b308e6d --- /dev/null +++ b/dep/dep.py @@ -0,0 +1,488 @@ +#!/usr/bin/python2.7 +""" +Dependency manager for llvm CI builds. + +We have complex dependencies for some of our CI builds. This will serve +as a system to help document and enforce them. + +Developer notes: + +- We are trying to keep package dependencies to a minimum in this project. So it +does not require an installer. It should be able to be run as a stand alone script +when checked out of VCS. So, don't import anything not in the Python 2.7 +standard library. + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import argparse +import json +import platform +import re +import subprocess + + +try: + from typing import List, Text, Union, Dict, Type, Optional +except ImportError as e: + Optional = Type = Dict = List = Text = Union = None + pass # Not really needed at runtime, so okay to not have installed. + + +VERSION = '0.1' +"""We have a built in version check, so we can require specific features and fixes.""" + + +class Version(object): + """Model a version number, which can be compared to another version number. + + Keeps a nice looking text version around as well for printing. + + This abstraction exists to make some of the more complex comparisons easier, + as well as collecting and printing versions. + + In the future, we might want to have some different comparison, + for instance, 4.0 == 4.0.0 -> True. + + """ + + def __init__(self, text): + """Create a version from a . separated version string.""" + self.text = text + self.numeric = [int(d) for d in text.split(".")] + + def __gt__(self, other): + """Compare the numeric representation of the version.""" + return self.numeric.__gt__(other.numeric) + + def __lt__(self, other): + """Compare the numeric representation of the version.""" + return self.numeric.__lt__(other.numeric) + + def __eq__(self, other): + """Compare the numeric representation of the version.""" + return self.numeric.__eq__(other.numeric) + + def __le__(self, other): + """Compare the numeric representation of the version.""" + return self.numeric.__le__(other.numeric) + + def __ge__(self, other): + """Compare the numeric representation of the version.""" + return self.numeric.__ge__(other.numeric) + + def __repr__(self): + """Print the original text representation of the Version.""" + return "v{}".format(self.text) + + +class Dependency(object): + """Dependency Abstract base class.""" + + def __init__(self, line, str_kind): + """Save line information. + + :param line: A parsed Line object that contains the raw dependency deceleration. + :param str_kind: The determined kind of the Dependency. + """ + # type: (Line, Text) -> object + self.line = line + self.str_kind = str_kind + self.installed_version = None + + def parse(self): + """Read the input line and prepare to verify this dependency. + + Raise a MalformedDependencyError if three is something wrong. + + Should return nothing, but get the dependency ready for verification. + """ + raise NotImplementedError() + + def verify(self): + # type: () -> bool + """Determine if this dependency met. + + :returns: True when the dependency is met, otherwise False. + """ + raise NotImplementedError() + + def inject(self): + """If possible, modify the system to meet the dependency.""" + raise NotImplementedError() + + def verify_and_act(self): + """Parse, then verify and trigger pass or fail. + + Extract that out here, so we don't duplicate the logic in the subclasses. + """ + met = self.verify() + if met: + self.verify_pass() + else: + self.verify_fail() + + def verify_fail(self): + """When dependency is not met, raise an exception. + + This is the default behavior; but I want the subclasses to be able + to change it. + """ + raise MissingDependencyError(self, self.installed_version) + + def verify_pass(self): + """Print a nice message that the dependency is met. + + I'm not sure we even want to print this, but we will for now. It might + be to verbose. Subclasses should override this if wanted. + """ + print("Dependency met", str(self)) + + +class MalformedDependency(Exception): + """Raised when parsing a dependency directive fails. + + This is situations like the regexes not matching, or part of the dependency directive missing. + + Should probably record more useful stuff, but Exception.message is set. So we can print it later. + """ + + pass + + +def brew_cmd(command): + # type: (List[Text]) -> Dict[Text, object] + """Help run a brew command, and parse the output. + + Brew has a json output option which we use. Run the command and parse the stdout + as json and return the result. + :param command: The brew command to execute, and parse the output of. + :return: + """ + assert "--json=v1" in command, "Must pass JSON arg so we can parse the output." + out = subprocess.check_output(command) + brew_info = json.loads(out) + return brew_info + + +class MissingDependencyError(Exception): + """Fail verification with one of these when we determine a dependency is not met. + + For each dependency, we will print a useful message with the dependency line as well as the + reason it was not matched. + """ + + def __init__(self, dependency, installed=None): + # type: (Dependency, Optional[Text]) -> None + """Raise when a dependency is not met. + + This exception can be printed as the error message. + + :param dependency: The dependency that is not being met. + :param installed: what was found to be installed instead. + """ + # type: (Dependency, Union[Text, Version]) -> None + super(MissingDependencyError, self).__init__() + self.dependency = dependency + self.installed = installed + + def __str__(self): + """For now, we will just print these as our error message.""" + return "missing dependency: {}, found {} installed, requested from {}".format(self.dependency, + self.installed, + self.dependency.line) + + +def check_version(installed, operator, requested): + """Check that the installed version does the operator of the requested. + + :param installed: The installed Version of the requirement. + :param operator: the text operator (==, <=, >=) + :param requested: The requested Version of the requirement. + :return: True if the requirement is satisfied. + """ + # type: (Version, Text, Version) -> bool + + dependency_met = False + if operator == "==" and installed == requested: + dependency_met = True + if operator == "<=" and installed <= requested: + dependency_met = True + if operator == ">=" and installed >= requested: + dependency_met = True + return dependency_met + + +class ConMan(Dependency): + """Version self-check of this tool. + + In case we introduce something in the future, the dep files can + be made to depend on a specific version of this tool. We will + increment the versions manually. + + """ + + conman_re = re.compile(r'(?P<command>\w+)\s+(?P<operator>>=|<=|==)\s*(?P<version_text>[\d.-_]+)') + """For example: config_manager <= 0.1""" + + def __init__(self, line, kind): + """Check that this tool is up to date.""" + super(ConMan, self).__init__(line, kind) + self.command = None + self.operator = None + self.version = None + self.version_text = None + self.installed_version = None + + def parse(self): + """Parse dependency.""" + text = self.line.text + match = self.conman_re.match(text) + if not match: + raise MalformedDependency("Expression does not compile in {}: {}".format(self.__class__.__name__, + self.line)) + self.__dict__.update(match.groupdict()) + + self.version = Version(self.version_text) + + def verify(self): + """Verify the version of this tool.""" + self.installed_version = Version(VERSION) + + return check_version(self.installed_version, self.operator, self.version) + + def inject(self): + """Can't really do much here.""" + pass + + def __str__(self): + """Show as dependency and version.""" + return "{} {}".format(self.str_kind, self.version) + + +class HostOSVersion(Dependency): + """Use Python's platform module to get host OS information and verify. + + Wew can only verify, but not inject for host OS version. + """ + + conman_re = re.compile(r'(?P<command>\w+)\s+(?P<operator>>=|<=|==)\s*(?P<version_text>[\d.-_]+)') + + def __init__(self, line, kind): + """Parse and Verify host OS version using Python's platform module. + + :param line: Line with teh Dependencies deceleration. + :param kind: the dependency kind that was detected by the parser. + """ + # type: (Line, Text) -> None + super(HostOSVersion, self).__init__(line, kind) + self.command = None + self.operator = None + self.version = None + self.version_text = None + self.installed_version = None + + def parse(self): + """Parse dependency.""" + text = self.line.text + match = self.conman_re.match(text) + if not match: + raise MalformedDependency("Expression does not compile in {}: {}".format(self.__class__.__name__, + self.line)) + self.__dict__.update(match.groupdict()) + + self.version = Version(self.version_text) + + def verify(self): + """Verify the request host OS version holds.""" + self.installed_version = Version(platform.mac_ver()[0]) + + return check_version(self.installed_version, self.operator, self.version) + + def inject(self): + """Can't change the host OS version, so not much to do here.""" + pass + + def __str__(self): + """For printing in error messages.""" + return "{} {}".format(self.str_kind, self.version) + + +class Brew(Dependency): + """Verify and Inject brew package dependencies.""" + + # brew <package> <operator> <version>. Operator may not have spaces around it. + brew_re = re.compile(r'(?P<command>\w+)\s+(?P<package>\w+)\s*(?P<operator>>=|<=|==)\s*(?P<version_text>[\d.-_]+)') + + def __init__(self, line, kind): + # type: (Line, Text) -> None + """Parse and verify brew package is installed. + + :param line: the Line with the deceleration of the dependency. + :param kind: the detected dependency kind. + """ + super(Brew, self).__init__(line, kind) + self.command = None + self.operator = None + self.package = None + self.version = None + self.version_text = None + self.installed_version = None + + def parse(self): + """Parse this dependency.""" + text = self.line.text + match = self.brew_re.match(text) + if not match: + raise MalformedDependency("Expression does not compile in {}: {}".format(self.__class__.__name__, + self.line)) + self.__dict__.update(match.groupdict()) + + self.version = Version(self.version_text) + + def verify(self): + """Verify the packages in brew match this dependency.""" + brew_package_config = brew_cmd(['/usr/local/bin/brew', 'info', self.package, "--json=v1"]) + version = None + for brew_package in brew_package_config: + name = brew_package['name'] + install_info = brew_package.get('installed') + for versions in install_info: + version = versions['version'] if versions else None + if name == self.package: + break + if not version: + # The package is not installed at all. + raise MissingDependencyError(self, "nothing") + self.installed_version = Version(version) + return check_version(self.installed_version, self.operator, self.version) + + def inject(self): + """Not implemented.""" + raise NotImplementedError() + + def __str__(self): + """Dependency kind, package and version, for printing in error messages.""" + return "{} {} {}".format(self.str_kind, self.package, self.version) + + +dependencies_implementations = {'brew': Brew, + 'os_version': HostOSVersion, + 'config_manager': ConMan} + + +def dependency_factory(line): + """Given a line, create a concrete dependency for it. + + :param line: The line with the dependency info + :return: Some subclass of Dependency, based on what was in the line. + """ + # type: Text -> Dependency + kind = line.text.split()[0] + try: + return dependencies_implementations[kind](line, kind) + except KeyError: + raise MalformedDependency("Don't know about {} kind of dependency.".format(kind)) + + +class Line(object): + """A preprocessed line. Understands file and line number as well as comments.""" + + def __init__(self, filename, line_number, text, comment): + # type: (Text, int, Text, Text) -> None + """Raw Line information, split into the dependency deceleration and comment. + + :param filename: the input filename. + :param line_number: the line number in the input file. + :param text: Non-comment part of the line. + :param comment: Text from the comment part of the line if any. + """ + self.filename = filename + self.line_number = line_number + self.text = text + self.comment = comment + + def __repr__(self): + """Reconstruct the line for pretty printing.""" + return "{}:{}: {}{}".format(self.filename, + self.line_number, + self.text, + " # " + self.comment if self.comment else "") + + +# For stripping comments out of lines. +comment_re = re.compile(r'(?P<main_text>.*)#(?P<comment>.*)') + + +# noinspection PyUnresolvedReferences +def _parse_dep_file(lines, filename): + # type: (List[Text], Text) -> List[Line] + process_lines = [] + for num, text in enumerate(lines): + if "#" in text: + bits = comment_re.match(text) + main_text = bits.groupdict().get('main_text') + comment = bits.groupdict().get('comment') + else: + main_text = text + comment = None + if main_text: + main_text = main_text.strip() + if comment: + comment = comment.strip() + process_lines.append(Line(filename, num, main_text, comment)) + + return process_lines + + +def parse_dependencies(file_names): + """Program logic: read files, verify dependencies. + + For each input file, read lines and create dependencies. Verify each dependency. + + :param file_names: files to read dependencies from. + :return: The list of dependencies, each verified. + """ + # type: (List[Text]) -> List[Type[Dependency]] + preprocessed_lines = [] + for file_name in file_names: + with open(file_name, 'r') as f: + lines = f.readlines() + preprocessed_lines.extend(_parse_dep_file(lines, file_name)) + + dependencies = [dependency_factory(l) for l in preprocessed_lines if l.text] + [d.parse() for d in dependencies] + for d in dependencies: + try: + met = d.verify() + if met: + d.verify_pass() + else: + d.verify_fail() + + except MissingDependencyError as exec_info: + print("Error:", exec_info) + + return dependencies + + +def main(): + """Parse arguments and trigger dependency verification.""" + parser = argparse.ArgumentParser(description='Verify and install dependencies.') + parser.add_argument('command', help="What to do.") + + parser.add_argument('dependencies', nargs='+', help="Path to dependency files.") + + args = parser.parse_args() + + parse_dependencies(args.dependencies) + + return True + + +if __name__ == '__main__': + main() |