From f0d9e8820c62604621b22d3a83dea599a6b948b3 Mon Sep 17 00:00:00 2001 From: Chris Matthews Date: Thu, 11 Jan 2018 23:55:23 +0000 Subject: Simple dependency manager for our CI jobs We have some pretty complex dependencies in our CI jobs. These scripts are an attempt to have a more systematic approach to checking those dependencies. git-svn-id: https://llvm.org/svn/llvm-project/zorg/trunk@322331 91177308-0d34-0410-b5e6-96231b3b80d8 --- dep/dep.py | 488 +++++++++++++++++++++++++ dep/requirements.txt | 2 + dep/tests/Dependencies0 | 2 + dep/tests/Dependencies1 | 17 + dep/tests/assets/brew_cmake_installed.json | 76 ++++ dep/tests/assets/brew_ninja_not_installed.json | 58 +++ dep/tests/conftest.py | 11 + dep/tests/test_dep.py | 134 +++++++ 8 files changed, 788 insertions(+) create mode 100644 dep/dep.py create mode 100644 dep/requirements.txt create mode 100644 dep/tests/Dependencies0 create mode 100644 dep/tests/Dependencies1 create mode 100644 dep/tests/assets/brew_cmake_installed.json create mode 100644 dep/tests/assets/brew_ninja_not_installed.json create mode 100644 dep/tests/conftest.py create mode 100644 dep/tests/test_dep.py diff --git a/dep/dep.py b/dep/dep.py new file mode 100644 index 00000000..6b308e6d --- /dev/null +++ b/dep/dep.py @@ -0,0 +1,488 @@ +#!/usr/bin/python2.7 +""" +Dependency manager for llvm CI builds. + +We have complex dependencies for some of our CI builds. This will serve +as a system to help document and enforce them. + +Developer notes: + +- We are trying to keep package dependencies to a minimum in this project. So it +does not require an installer. It should be able to be run as a stand alone script +when checked out of VCS. So, don't import anything not in the Python 2.7 +standard library. + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import argparse +import json +import platform +import re +import subprocess + + +try: + from typing import List, Text, Union, Dict, Type, Optional +except ImportError as e: + Optional = Type = Dict = List = Text = Union = None + pass # Not really needed at runtime, so okay to not have installed. + + +VERSION = '0.1' +"""We have a built in version check, so we can require specific features and fixes.""" + + +class Version(object): + """Model a version number, which can be compared to another version number. + + Keeps a nice looking text version around as well for printing. + + This abstraction exists to make some of the more complex comparisons easier, + as well as collecting and printing versions. + + In the future, we might want to have some different comparison, + for instance, 4.0 == 4.0.0 -> True. + + """ + + def __init__(self, text): + """Create a version from a . separated version string.""" + self.text = text + self.numeric = [int(d) for d in text.split(".")] + + def __gt__(self, other): + """Compare the numeric representation of the version.""" + return self.numeric.__gt__(other.numeric) + + def __lt__(self, other): + """Compare the numeric representation of the version.""" + return self.numeric.__lt__(other.numeric) + + def __eq__(self, other): + """Compare the numeric representation of the version.""" + return self.numeric.__eq__(other.numeric) + + def __le__(self, other): + """Compare the numeric representation of the version.""" + return self.numeric.__le__(other.numeric) + + def __ge__(self, other): + """Compare the numeric representation of the version.""" + return self.numeric.__ge__(other.numeric) + + def __repr__(self): + """Print the original text representation of the Version.""" + return "v{}".format(self.text) + + +class Dependency(object): + """Dependency Abstract base class.""" + + def __init__(self, line, str_kind): + """Save line information. + + :param line: A parsed Line object that contains the raw dependency deceleration. + :param str_kind: The determined kind of the Dependency. + """ + # type: (Line, Text) -> object + self.line = line + self.str_kind = str_kind + self.installed_version = None + + def parse(self): + """Read the input line and prepare to verify this dependency. + + Raise a MalformedDependencyError if three is something wrong. + + Should return nothing, but get the dependency ready for verification. + """ + raise NotImplementedError() + + def verify(self): + # type: () -> bool + """Determine if this dependency met. + + :returns: True when the dependency is met, otherwise False. + """ + raise NotImplementedError() + + def inject(self): + """If possible, modify the system to meet the dependency.""" + raise NotImplementedError() + + def verify_and_act(self): + """Parse, then verify and trigger pass or fail. + + Extract that out here, so we don't duplicate the logic in the subclasses. + """ + met = self.verify() + if met: + self.verify_pass() + else: + self.verify_fail() + + def verify_fail(self): + """When dependency is not met, raise an exception. + + This is the default behavior; but I want the subclasses to be able + to change it. + """ + raise MissingDependencyError(self, self.installed_version) + + def verify_pass(self): + """Print a nice message that the dependency is met. + + I'm not sure we even want to print this, but we will for now. It might + be to verbose. Subclasses should override this if wanted. + """ + print("Dependency met", str(self)) + + +class MalformedDependency(Exception): + """Raised when parsing a dependency directive fails. + + This is situations like the regexes not matching, or part of the dependency directive missing. + + Should probably record more useful stuff, but Exception.message is set. So we can print it later. + """ + + pass + + +def brew_cmd(command): + # type: (List[Text]) -> Dict[Text, object] + """Help run a brew command, and parse the output. + + Brew has a json output option which we use. Run the command and parse the stdout + as json and return the result. + :param command: The brew command to execute, and parse the output of. + :return: + """ + assert "--json=v1" in command, "Must pass JSON arg so we can parse the output." + out = subprocess.check_output(command) + brew_info = json.loads(out) + return brew_info + + +class MissingDependencyError(Exception): + """Fail verification with one of these when we determine a dependency is not met. + + For each dependency, we will print a useful message with the dependency line as well as the + reason it was not matched. + """ + + def __init__(self, dependency, installed=None): + # type: (Dependency, Optional[Text]) -> None + """Raise when a dependency is not met. + + This exception can be printed as the error message. + + :param dependency: The dependency that is not being met. + :param installed: what was found to be installed instead. + """ + # type: (Dependency, Union[Text, Version]) -> None + super(MissingDependencyError, self).__init__() + self.dependency = dependency + self.installed = installed + + def __str__(self): + """For now, we will just print these as our error message.""" + return "missing dependency: {}, found {} installed, requested from {}".format(self.dependency, + self.installed, + self.dependency.line) + + +def check_version(installed, operator, requested): + """Check that the installed version does the operator of the requested. + + :param installed: The installed Version of the requirement. + :param operator: the text operator (==, <=, >=) + :param requested: The requested Version of the requirement. + :return: True if the requirement is satisfied. + """ + # type: (Version, Text, Version) -> bool + + dependency_met = False + if operator == "==" and installed == requested: + dependency_met = True + if operator == "<=" and installed <= requested: + dependency_met = True + if operator == ">=" and installed >= requested: + dependency_met = True + return dependency_met + + +class ConMan(Dependency): + """Version self-check of this tool. + + In case we introduce something in the future, the dep files can + be made to depend on a specific version of this tool. We will + increment the versions manually. + + """ + + conman_re = re.compile(r'(?P\w+)\s+(?P>=|<=|==)\s*(?P[\d.-_]+)') + """For example: config_manager <= 0.1""" + + def __init__(self, line, kind): + """Check that this tool is up to date.""" + super(ConMan, self).__init__(line, kind) + self.command = None + self.operator = None + self.version = None + self.version_text = None + self.installed_version = None + + def parse(self): + """Parse dependency.""" + text = self.line.text + match = self.conman_re.match(text) + if not match: + raise MalformedDependency("Expression does not compile in {}: {}".format(self.__class__.__name__, + self.line)) + self.__dict__.update(match.groupdict()) + + self.version = Version(self.version_text) + + def verify(self): + """Verify the version of this tool.""" + self.installed_version = Version(VERSION) + + return check_version(self.installed_version, self.operator, self.version) + + def inject(self): + """Can't really do much here.""" + pass + + def __str__(self): + """Show as dependency and version.""" + return "{} {}".format(self.str_kind, self.version) + + +class HostOSVersion(Dependency): + """Use Python's platform module to get host OS information and verify. + + Wew can only verify, but not inject for host OS version. + """ + + conman_re = re.compile(r'(?P\w+)\s+(?P>=|<=|==)\s*(?P[\d.-_]+)') + + def __init__(self, line, kind): + """Parse and Verify host OS version using Python's platform module. + + :param line: Line with teh Dependencies deceleration. + :param kind: the dependency kind that was detected by the parser. + """ + # type: (Line, Text) -> None + super(HostOSVersion, self).__init__(line, kind) + self.command = None + self.operator = None + self.version = None + self.version_text = None + self.installed_version = None + + def parse(self): + """Parse dependency.""" + text = self.line.text + match = self.conman_re.match(text) + if not match: + raise MalformedDependency("Expression does not compile in {}: {}".format(self.__class__.__name__, + self.line)) + self.__dict__.update(match.groupdict()) + + self.version = Version(self.version_text) + + def verify(self): + """Verify the request host OS version holds.""" + self.installed_version = Version(platform.mac_ver()[0]) + + return check_version(self.installed_version, self.operator, self.version) + + def inject(self): + """Can't change the host OS version, so not much to do here.""" + pass + + def __str__(self): + """For printing in error messages.""" + return "{} {}".format(self.str_kind, self.version) + + +class Brew(Dependency): + """Verify and Inject brew package dependencies.""" + + # brew . Operator may not have spaces around it. + brew_re = re.compile(r'(?P\w+)\s+(?P\w+)\s*(?P>=|<=|==)\s*(?P[\d.-_]+)') + + def __init__(self, line, kind): + # type: (Line, Text) -> None + """Parse and verify brew package is installed. + + :param line: the Line with the deceleration of the dependency. + :param kind: the detected dependency kind. + """ + super(Brew, self).__init__(line, kind) + self.command = None + self.operator = None + self.package = None + self.version = None + self.version_text = None + self.installed_version = None + + def parse(self): + """Parse this dependency.""" + text = self.line.text + match = self.brew_re.match(text) + if not match: + raise MalformedDependency("Expression does not compile in {}: {}".format(self.__class__.__name__, + self.line)) + self.__dict__.update(match.groupdict()) + + self.version = Version(self.version_text) + + def verify(self): + """Verify the packages in brew match this dependency.""" + brew_package_config = brew_cmd(['/usr/local/bin/brew', 'info', self.package, "--json=v1"]) + version = None + for brew_package in brew_package_config: + name = brew_package['name'] + install_info = brew_package.get('installed') + for versions in install_info: + version = versions['version'] if versions else None + if name == self.package: + break + if not version: + # The package is not installed at all. + raise MissingDependencyError(self, "nothing") + self.installed_version = Version(version) + return check_version(self.installed_version, self.operator, self.version) + + def inject(self): + """Not implemented.""" + raise NotImplementedError() + + def __str__(self): + """Dependency kind, package and version, for printing in error messages.""" + return "{} {} {}".format(self.str_kind, self.package, self.version) + + +dependencies_implementations = {'brew': Brew, + 'os_version': HostOSVersion, + 'config_manager': ConMan} + + +def dependency_factory(line): + """Given a line, create a concrete dependency for it. + + :param line: The line with the dependency info + :return: Some subclass of Dependency, based on what was in the line. + """ + # type: Text -> Dependency + kind = line.text.split()[0] + try: + return dependencies_implementations[kind](line, kind) + except KeyError: + raise MalformedDependency("Don't know about {} kind of dependency.".format(kind)) + + +class Line(object): + """A preprocessed line. Understands file and line number as well as comments.""" + + def __init__(self, filename, line_number, text, comment): + # type: (Text, int, Text, Text) -> None + """Raw Line information, split into the dependency deceleration and comment. + + :param filename: the input filename. + :param line_number: the line number in the input file. + :param text: Non-comment part of the line. + :param comment: Text from the comment part of the line if any. + """ + self.filename = filename + self.line_number = line_number + self.text = text + self.comment = comment + + def __repr__(self): + """Reconstruct the line for pretty printing.""" + return "{}:{}: {}{}".format(self.filename, + self.line_number, + self.text, + " # " + self.comment if self.comment else "") + + +# For stripping comments out of lines. +comment_re = re.compile(r'(?P.*)#(?P.*)') + + +# noinspection PyUnresolvedReferences +def _parse_dep_file(lines, filename): + # type: (List[Text], Text) -> List[Line] + process_lines = [] + for num, text in enumerate(lines): + if "#" in text: + bits = comment_re.match(text) + main_text = bits.groupdict().get('main_text') + comment = bits.groupdict().get('comment') + else: + main_text = text + comment = None + if main_text: + main_text = main_text.strip() + if comment: + comment = comment.strip() + process_lines.append(Line(filename, num, main_text, comment)) + + return process_lines + + +def parse_dependencies(file_names): + """Program logic: read files, verify dependencies. + + For each input file, read lines and create dependencies. Verify each dependency. + + :param file_names: files to read dependencies from. + :return: The list of dependencies, each verified. + """ + # type: (List[Text]) -> List[Type[Dependency]] + preprocessed_lines = [] + for file_name in file_names: + with open(file_name, 'r') as f: + lines = f.readlines() + preprocessed_lines.extend(_parse_dep_file(lines, file_name)) + + dependencies = [dependency_factory(l) for l in preprocessed_lines if l.text] + [d.parse() for d in dependencies] + for d in dependencies: + try: + met = d.verify() + if met: + d.verify_pass() + else: + d.verify_fail() + + except MissingDependencyError as exec_info: + print("Error:", exec_info) + + return dependencies + + +def main(): + """Parse arguments and trigger dependency verification.""" + parser = argparse.ArgumentParser(description='Verify and install dependencies.') + parser.add_argument('command', help="What to do.") + + parser.add_argument('dependencies', nargs='+', help="Path to dependency files.") + + args = parser.parse_args() + + parse_dependencies(args.dependencies) + + return True + + +if __name__ == '__main__': + main() diff --git a/dep/requirements.txt b/dep/requirements.txt new file mode 100644 index 00000000..c6ed6caf --- /dev/null +++ b/dep/requirements.txt @@ -0,0 +1,2 @@ +pytest +typing \ No newline at end of file diff --git a/dep/tests/Dependencies0 b/dep/tests/Dependencies0 new file mode 100644 index 00000000..a6fa8325 --- /dev/null +++ b/dep/tests/Dependencies0 @@ -0,0 +1,2 @@ +# This is a test +brew cmake >= 3.10 \ No newline at end of file diff --git a/dep/tests/Dependencies1 b/dep/tests/Dependencies1 new file mode 100644 index 00000000..30dd6163 --- /dev/null +++ b/dep/tests/Dependencies1 @@ -0,0 +1,17 @@ +# We build in an explicit config manager version check, since it will not be deployed in +config_manager >= 0.1 + +# Also get more includes from: +include ./some_other_file +# Use the following sources for installing. +source brew http://git.llvm.org/git/some-cask.git +source pip https://pypi.llvm.org +source root https://llvm.org/resources/ +# Dependenceis. +brew scons >= 3.0.1 +brew_cask Filecheck >= 5.0 + +os_version == 10.11.6 +pythontool lnt https://git.llvm.org/git/lnt.git + +device_ssh \ No newline at end of file diff --git a/dep/tests/assets/brew_cmake_installed.json b/dep/tests/assets/brew_cmake_installed.json new file mode 100644 index 00000000..2fa83939 --- /dev/null +++ b/dep/tests/assets/brew_cmake_installed.json @@ -0,0 +1,76 @@ +[ + { + "name": "cmake", + "full_name": "cmake", + "desc": "Cross-platform make", + "homepage": "https://www.cmake.org/", + "oldname": null, + "aliases": [], + "versions": { + "stable": "3.10.0", + "bottle": true, + "devel": null, + "head": "HEAD" + }, + "revision": 0, + "version_scheme": 0, + "installed": [ + { + "version": "3.10.0", + "used_options": [], + "built_as_bottle": true, + "poured_from_bottle": true, + "runtime_dependencies": [], + "installed_as_dependency": false, + "installed_on_request": true + } + ], + "linked_keg": "3.10.0", + "pinned": false, + "outdated": false, + "keg_only": false, + "dependencies": [ + "sphinx-doc" + ], + "recommended_dependencies": [], + "optional_dependencies": [], + "build_dependencies": [ + "sphinx-doc" + ], + "conflicts_with": [], + "caveats": null, + "requirements": [], + "options": [ + { + "option": "--without-docs", + "description": "Don't build man pages" + }, + { + "option": "--with-completion", + "description": "Install Bash completion (Has potential problems with system bash)" + } + ], + "bottle": { + "stable": { + "rebuild": 1, + "cellar": ":any_skip_relocation", + "prefix": "/usr/local", + "root_url": "https://homebrew.bintray.com/bottles", + "files": { + "high_sierra": { + "url": "https://homebrew.bintray.com/bottles/cmake-3.10.0.high_sierra.bottle.1.tar.gz", + "sha256": "fa4888d1d009e32398d0ec312b641f86f6eac53cdfd13e5dae57c07922c8033c" + }, + "sierra": { + "url": "https://homebrew.bintray.com/bottles/cmake-3.10.0.sierra.bottle.1.tar.gz", + "sha256": "5a6c5af53ce59a89d3f31880fdcc169359ec6ad49daa78ebcaf333c32f481590" + }, + "el_capitan": { + "url": "https://homebrew.bintray.com/bottles/cmake-3.10.0.el_capitan.bottle.1.tar.gz", + "sha256": "5e1d7d0abd668e008a695f51778d52b06a229ba6fef5014397f8dab9e4578eca" + } + } + } + } + } +] diff --git a/dep/tests/assets/brew_ninja_not_installed.json b/dep/tests/assets/brew_ninja_not_installed.json new file mode 100644 index 00000000..76b82480 --- /dev/null +++ b/dep/tests/assets/brew_ninja_not_installed.json @@ -0,0 +1,58 @@ +[ + { + "name": "ninja", + "full_name": "ninja", + "desc": "Small build system for use with gyp or CMake", + "homepage": "https://ninja-build.org/", + "oldname": null, + "aliases": [], + "versions": { + "stable": "1.8.2", + "bottle": true, + "devel": null, + "head": "HEAD" + }, + "revision": 0, + "version_scheme": 0, + "installed": [], + "linked_keg": null, + "pinned": false, + "outdated": false, + "keg_only": false, + "dependencies": [], + "recommended_dependencies": [], + "optional_dependencies": [], + "build_dependencies": [], + "conflicts_with": [], + "caveats": null, + "requirements": [], + "options": [ + { + "option": "--without-test", + "description": "Don't run build-time tests" + } + ], + "bottle": { + "stable": { + "rebuild": 0, + "cellar": ":any_skip_relocation", + "prefix": "/usr/local", + "root_url": "https://homebrew.bintray.com/bottles", + "files": { + "high_sierra": { + "url": "https://homebrew.bintray.com/bottles/ninja-1.8.2.high_sierra.bottle.tar.gz", + "sha256": "eeba4fff08b3ed4b308250fb650f7d06630acd18465900ba0e27cecfe925a6cc" + }, + "sierra": { + "url": "https://homebrew.bintray.com/bottles/ninja-1.8.2.sierra.bottle.tar.gz", + "sha256": "90ecf90948f0fa65c82011d79338d7c5ca2a4d0cb7cb8dc3892243f749fbe2eb" + }, + "el_capitan": { + "url": "https://homebrew.bintray.com/bottles/ninja-1.8.2.el_capitan.bottle.tar.gz", + "sha256": "675165ce642fa811e1a0a363be0ba66a7b907d46056f89fd20938aa33e7d59f7" + } + } + } + } + } +] diff --git a/dep/tests/conftest.py b/dep/tests/conftest.py new file mode 100644 index 00000000..33cd83b2 --- /dev/null +++ b/dep/tests/conftest.py @@ -0,0 +1,11 @@ +import pytest + + + + +@pytest.fixture +def stubargs(): + class TestArgs(object): + dependencies = ['./Dependencies0'] + command = 'verify' + return TestArgs() \ No newline at end of file diff --git a/dep/tests/test_dep.py b/dep/tests/test_dep.py new file mode 100644 index 00000000..5c877c4d --- /dev/null +++ b/dep/tests/test_dep.py @@ -0,0 +1,134 @@ +"""Test deps. + +Testing adds more requirements, as we use Pytest. That is not needed for running though. + +""" +import json +import os +import sys +import pytest + +import dep +from dep import Line, Brew, Version, MissingDependencyError, ConMan, HostOSVersion + +here = os.path.dirname(os.path.realpath(__file__)) + + +def test_simple_brew(): + """End-to-end test of a simple brew dependency.""" + dep.parse_dependencies(['./tests/Dependencies0']) + + +def test_main(): + """End-to-end test of larger dependency file.""" + sys.argv = ['/bin/deps', 'verify', './tests/Dependencies1'] + with pytest.raises(dep.MalformedDependency): + dep.main() + + +def test_brew_cmake_requirement(mocker): + """Detailed check of a brew cmake dependency.""" + line = Line("foo.c", 10, "brew cmake <= 3.10.0", "test") + + b = Brew(line, "brew") + b.parse() + assert b.operator == "<=" + assert b.command == "brew" + assert b.package == "cmake" + assert b.version_text == "3.10.0" + mocker.patch('dep.brew_cmd') + dep.brew_cmd.return_value = json.load(open(here + '/assets/brew_cmake_installed.json')) + b.verify_and_act() + assert dep.brew_cmd.called + + +def test_brew_ninja_not_installed_requirement(mocker): + """Detailed check of a unmatched brew requirement.""" + line = Line("foo.c", 11, "brew ninja <= 1.8.2", "We use ninja as clang's build system.") + + b = Brew(line, "brew") + b.parse() + assert b.operator == "<=" + assert b.command == "brew" + assert b.package == "ninja" + assert b.version_text == "1.8.2" + mocker.patch('dep.brew_cmd') + dep.brew_cmd.return_value = json.load(open(here + '/assets/brew_ninja_not_installed.json')) + # The package is not installed + with pytest.raises(MissingDependencyError) as exception_info: + b.verify_and_act() + + assert "missing dependency: brew ninja v1.8.2, found nothing installed" in str(exception_info) + assert dep.brew_cmd.called + + +def test_versions(): + """Unittests for the version comparison objects.""" + v1 = Version("3.2.1") + v2 = Version("3.3.1") + v3 = Version("3.2") + v4 = Version("3.2") + + # Check the values are parsed correctly. + assert v1.text == "3.2.1" + assert v1.numeric == [3, 2, 1] + assert v3.text == "3.2" + assert v3.numeric == [3, 2] + + # Check the operators work correctly. + assert v2 > v1 + assert v1 < v2 + assert v2 >= v1 + assert v1 <= v2 + assert v3 == v4 + + # Check that versions with different number of digits compare correctly. + assert v2 > v3 + assert v3 < v2 + + # TODO fix different digit comparisons. + # assert v4 == v1 + assert v3 >= v4 + + +def test_self_version_requirement(): + """Unittest of the self version check.""" + line = Line("foo.c", 10, "config_manager <= 0.1", "test") + + b = ConMan(line, "config_manager") + b.parse() + assert b.operator == "<=" + assert b.command == "config_manager" + assert b.version_text == "0.1" + + b.verify_and_act() + + line = Line("foo.c", 10, "config_manager <= 0.0.1", "test") + bad = ConMan(line, "config_manager") + bad.parse() + with pytest.raises(MissingDependencyError): + bad.verify_and_act() + line = Line("foo.c", 10, "config_manager == " + dep.VERSION, "test") + good = ConMan(line, "config_manager") + good.parse() + good.verify_and_act() + + +def test_host_os_version_requirement(mocker): + """Unittest of the host os version check.""" + line = Line("foo.c", 11, "os_version == 10.13.2", "test") + mocker.patch('dep.platform.mac_ver') + dep.platform.mac_ver.return_value = ('10.13.2', "", "") + b = HostOSVersion(line, "os_version") + b.parse() + assert b.operator == "==" + assert b.command == "os_version" + assert b.version_text == "10.13.2" + + b.verify_and_act() + + line = Line("foo.c", 10, "os_version == 10.13.1", "test") + bad = HostOSVersion(line, "os_version") + bad.parse() + with pytest.raises(MissingDependencyError): + bad.verify_and_act() -- cgit v1.2.3