#!/usr/bin/env python # # Copyright (C) 2018, 2019 Collabora Limited # Author: Guillaume Tucker # # This module is free software; you can redistribute it and/or modify it under # the terms of the GNU Lesser General Public License as published by the Free # Software Foundation; either version 2.1 of the License, or (at your option) # any later version. # # This library is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more # details. # # You should have received a copy of the GNU Lesser General Public License # along with this library; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import argparse import os import subprocess import sys import time import kernelci.build import kernelci.configs # ----------------------------------------------------------------------------- # Standard arguments that can be used in sub-commands # class Args(object): config= { 'name': '--config', 'help': "Build config name", } variant = { 'name': '--variant', 'help': "Build config variant name", } build_env = { 'name': '--build-env', 'help': "Build environment name", } storage = { 'name': '--storage', 'help': "Storage URL", 'default': "https://storage.kernelci.org", 'required': False, } api = { 'name': '--api', 'help': "Backend API URL", 'default': "https://api.kernelci.org", 'required': False, } token = { 'name': '--token', 'help': "Backend API token", } commit = { 'name': '--commit', 'help': "Git commit checksum", } describe = { 'name': '--describe', 'help': "Git describe", } describe_verbose = { 'name': '--describe-verbose', 'help': "Verbose version of git describe", } tree_name = { 'name': '--tree-name', 'help': "Name of a kernel tree", } tree_url = { 'name': '--tree-url', 'help': "URL of a kernel tree", } branch = { 'name': '--branch', 'help': "Name of a kernel branch in a tree", } mirror = { 'name': '--mirror', 'help': "Path to the local kernel git mirror", } kdir = { 'name': '--kdir', 'help': "Path to the kernel checkout directory", } arch = { 'name': '--arch', 'help': "CPU architecture name", } defconfig = { 'name': '--defconfig', 'help': "Kernel defconfig name", } j = { 'name': '-j', 'help': "Number of parallel build processes", } verbose = { 'name': '--verbose', 'help': "Verbose output", 'action': 'store_true', } output = { 'name': '--output', 'help': "Path the output directory", } json_path = { 'name': '--json-path', 'help': "Path to the JSON file", } # ----------------------------------------------------------------------------- # Commands # class Command(object): help = None args = None opt_args = None def __init__(self, sub_parser, name): if not self.help: raise AttributeError("Missing help message for {}".format(name)) self._parser = sub_parser.add_parser(name, help=self.help) if self.args: for arg in self.args: self._add_arg(arg, True) if self.opt_args: for arg in self.opt_args: self._add_arg(arg, False) self._parser.set_defaults(func=self) def __call__(self, *args, **kw): raise NotImplementedError("Command not implemented") def _add_arg(self, arg, required=True): kw = dict(arg) arg_name = kw.pop('name') if required: kw.setdefault('required', True) self._parser.add_argument(arg_name, **kw) class cmd_list_configs(Command): help = "List the build configurations" def __call__(self, configs, args): for conf_name in configs['build_configs'].keys(): print(conf_name) return True class cmd_check_new_commit(Command): help = "Check if a new commit is available on a branch" args = [Args.config, Args.storage] def __call__(self, configs, args): conf = configs['build_configs'][args.config] update = kernelci.build.check_new_commit(conf, args.storage) if update is False or update is True: return update print(update) return True class cmd_update_last_commit(Command): help = "Update the last commit file on the remote storage server" args = [Args.config, Args.api, Args.token, Args.commit] def __call__(self, configs, args): conf = configs['build_configs'][args.config] kernelci.build.set_last_commit(conf, args.api, args.token, args.commit) return True class cmd_tree_branch(Command): help = "Print the name of the tree and branch for a given build config" args = [Args.config] def __call__(self, configs, args): conf = configs['build_configs'][args.config] print(conf.tree.name) print(conf.tree.url) print(conf.branch) return True class cmd_update_mirror(Command): help = "Update the local kernel git mirror" args = [Args.config, Args.mirror] def __call__(self, configs, args): conf = configs['build_configs'][args.config] kernelci.build.update_mirror(conf, args.mirror) return True class cmd_update_repo(Command): help = "Update the local kernel repository checkout" args = [Args.config, Args.kdir] opt_args = [Args.mirror] def __call__(self, configs, args): conf = configs['build_configs'][args.config] kernelci.build.update_repo(conf, args.kdir, args.mirror) return True class cmd_describe(Command): help = "Print the git commit, describe and verbose describe from kdir" args = [Args.config, Args.kdir] def __call__(self, configs, args): conf = configs['build_configs'][args.config] commit = kernelci.build.head_commit(args.kdir) describe = kernelci.build.git_describe(conf.tree.name, args.kdir) verbose = kernelci.build.git_describe_verbose(args.kdir) print(commit) print(describe) print(verbose or describe) return True class cmd_generate_fragments(Command): help = "Generate the config fragment files in kdir" args = [Args.config, Args.kdir] def __call__(self, configs, args): conf = configs['build_configs'][args.config] kernelci.build.generate_fragments(conf, args.kdir) return True class cmd_generate_defconfig_fragments(Command): help = "Generate the config fragment files for a given defconfig" args = [Args.defconfig, Args.kdir] def __call__(self, configs, args): fragments = configs['fragments'] split = args.defconfig.split('+') defconfig = split.pop(0) for part in split: frag = fragments.get(part) if frag and frag.configs: kernelci.build.generate_config_fragment(frag, args.kdir) return True class cmd_expand_fragments(Command): help = "Expand a defconfig string with full fragment paths" args = [Args.defconfig] def __call__(self, configs, args): frags = configs['fragments'] split = args.defconfig.split('+') expanded = [split.pop(0)] for part in split: frag = frags.get(part) if frag: expanded.append(frag.path) else: expanded.append(part) print('+'.join(expanded)) return True class cmd_push_tarball(Command): help = "Create and up a source tarball to the remote storage server" args = [Args.config, Args.kdir, Args.storage, Args.api, Args.token] def __call__(self, configs, args): conf = configs['build_configs'][args.config] func_args = (conf, args.kdir, args.storage, args.api, args.token) if not all(func_args): print("Invalid arguments") return False tarball_url = kernelci.build.push_tarball(*func_args) if not tarball_url: return False print(tarball_url) return True class cmd_list_variants(Command): help = "Print the list of build variants for a given build configuration" args = [Args.config] def __call__(self, configs, args): conf = configs['build_configs'][args.config] for variant in conf.variants: print(variant.name) return True class cmd_arch_list(Command): help = "Print the list of CPU architecture names for a given build variant" args = [Args.config, Args.variant] def __call__(self, configs, args): conf = configs['build_configs'][args.config] variant = conf.get_variant(args.variant) for arch in variant.arch_list: print(arch) return True def _show_build_env(build_env, arch=None): print(build_env.name) print(build_env.cc) print(build_env.cc_version) if arch: print(build_env.get_arch_name(args.arch)) xc = build_env.get_cross_compile(args.arch) if xc: print(xc) class cmd_get_build_env(Command): help = "Print the build environment parameters for a given build variant" args = [Args.config, Args.variant] opt_args = [Args.arch] def __call__(self, configs, args): conf = configs['build_configs'][args.config] variant = conf.get_variant(args.variant) _show_build_env(variant.build_environment, args.arch) return True class cmd_show_build_env(Command): help = "Show parameters of a given build environment" args = [Args.build_env] opt_args = [Args.arch] def __call__(self, configs, args): build_env = configs['build_environments'][args.build_env] _show_build_env(build_env, args.arch) return True class cmd_list_kernel_configs(Command): help = "List the kernel configs to build for a given build configuration" args = [Args.config, Args.kdir] opt_args = [Args.variant, Args.arch] def __call__(self, configs, args): conf = configs['build_configs'][args.config] configs = kernelci.build.list_kernel_configs( conf, args.kdir, args.variant, args.arch) for item in configs: print(' '.join(item)) return True class cmd_build_kernel(Command): help = "Build a kernel" args = [Args.build_env, Args.arch, Args.kdir] opt_args = [Args.defconfig, Args.j, Args.verbose, Args.output] def __call__(self, configs, args): build_env = configs['build_environments'][args.build_env] return kernelci.build.build_kernel( build_env, args.kdir, args.arch, args.defconfig, args.j, args.verbose, args.output) class cmd_install_kernel(Command): help = "Install the kernel binaries and build.json locally" args = [Args.kdir] opt_args = [Args.config, Args.tree_name, Args.tree_url, Args.branch, Args.commit, Args.describe, Args.describe_verbose, Args.output] def __call__(self, configs, args): if args.config: conf = configs['build_configs'][args.config] tree_name = conf.tree.name tree_url = conf.tree.url branch = conf.branch else: tree_name = args.tree_name tree_url = args.tree_url branch = args.branch if not all((tree_name, tree_url, branch)): print("""\ Invalid arguments, either of these 2 sets are possible: --tree-name, --tree-url and --branch or --config (in which case the tree and branch are read from the YAML config)\ """) return False return kernelci.build.install_kernel( args.kdir, tree_name, tree_url, branch, args.commit, args.describe, args.describe_verbose, args.output) class cmd_push_kernel(Command): help = "Push the kernel binaries" args = [Args.kdir, Args.api, Args.token] def __call__(self, configs, args): return kernelci.build.push_kernel(args.kdir, args.api, args.token) class cmd_publish_kernel(Command): help = "Publish the kernel meta-data" args = [Args.kdir] opt_args = [Args.api, Args.token, Args.json_path] def __call__(self, configs, args): if not ((args.api and args.token) or args.json_path): print("""\ Invalid arguments, please provide at least one of these sets of options: --token, --api to publish to the backend server --json-path to save the data in a local JSON file\ """) return False return kernelci.build.publish_kernel( args.kdir, api=args.api, token=args.token, json_path=args.json_path) # ----------------------------------------------------------------------------- # Main # if __name__ == '__main__': parser = argparse.ArgumentParser("kci_build") parser.add_argument("--build-configs", default="build-configs.yaml", help="Path to build configs file") sub_parser = parser.add_subparsers(title="Commands", help="List of available commands") commands = dict() for k in globals().keys(): split = k.split('cmd_') if len(split) == 2: obj = globals().get(k) if issubclass(obj, Command): cmd_name = split[1] commands[cmd_name] = obj(sub_parser, cmd_name) args = parser.parse_args() configs = kernelci.configs.builds_from_yaml(args.build_configs) status = args.func(configs, args) sys.exit(0 if status is True else 1)