path: root/pre-built-images.py

#!/usr/bin/python

import hashlib
import os
import string
import subprocess
import time

from linaro_image_tools import cmd_runner


class HwPack:
    def __init__(self, lmc_name, eula=False, pre_inst_script=None):
        self.lmc_name = lmc_name
        if eula:
            self.eula = 'EULA.txt'
        else:
            self.eula = 'OPEN-EULA.txt'
        self.pre_inst_script = pre_inst_script

    def do_eula(self, image_file):
        fname = '%s/%s' % (os.path.dirname(image_file), self.eula)
        with file(fname, 'a'):
            pass  # just need the file to exist

# Mapping hwpack type - linaro-media-create --dev option
HWPACK_DEV = {
    'arndale': HwPack('arndale'),
    'arndale-be': HwPack('arndale'),
    'arndale-octa': HwPack('arndale-octa'),
    'beagleboard': HwPack('beagle'),
    'beaglebone': HwPack('beaglebone'),
    'efikamx': HwPack('efikamx'),
    'fastmodel': HwPack('fastmodel'),
    'highbank': HwPack('highbank'),
    'igep': HwPack('igep'),
    'leb-origen': HwPack('origen'),
    'lsk-vexpress': HwPack('vexpress'),
    'lsk-vexpress64': HwPack('vexpress'),
    'lt-mx5': HwPack('mx53loco'),
    'lt-mx6': HwPack('mx6qsabrelite'),
    'lt-origen': HwPack('origen'),
    'lt-panda': HwPack('panda'),
    'lt-panda-x11-base': HwPack('panda'),
    'lt-snowball': HwPack('snowball_sd', True, 'ste-preinstall.sh'),
    'lt-snowball-x11-base': HwPack('snowball_sd', True, 'ste-preinstall.sh'),
    'lt-vexpress': HwPack('vexpress'),
    'lt-vexpress64': HwPack('vexpress'),
    'midway': HwPack('highbank'),
    'origen': HwPack('origen'),
    'overo': HwPack('overo'),
    'panda': HwPack('panda'),
    'panda-be': HwPack('panda'),
    'snowball': HwPack('snowball_sd', True),
    'ti-panda-x11-base': HwPack('panda'),
    'vexpress': HwPack('vexpress'),
    'vexpress64': HwPack('vexpress'),
}

# Mapping of rootfs type - image size
ROOTFS_SIZE = {
    'alip': '2G',
    'developer': '1G',
    'gnome': '2G',
    'lamp-armv8': '2G',
    'leg-java-armv8': '2G',
    'linarotv-xbmc': '3G',
    'minimal-armv8': '512M',
    'nano': '1G',
    'nano-lava': '1G',
    'sdk-armv8': '2G',
    'server': '1G',
    'ubuntu-desktop': '3G',
}


def main():
    # Snapshots base URL
    snapshots_url = 'http://snapshots.linaro.org'

    # Name of the hardware pack project
    hwpack_job_name = os.environ.get('HWPACK_JOB_NAME')

    # Distribution, architecture and hardware pack type
    ret_split = hwpack_job_name.split('-', 2)
    (distribution, architecture, hwpack_type) =\
    ret_split[0], ret_split[1], ret_split[2]
    if hwpack_type.startswith('pre-built-images-'):
        hwpack_type = hwpack_type[(len('pre-built-images-')):]
    if hwpack_type.find('='):
        hwpack_type = hwpack_type[(len('pre-built-images/')):]
        ret_split = dict(
            token.split('=') for token in hwpack_type.split(','))
        hwpack_type = ret_split['hwpack']

    # Rootfs type, default is developer
    rootfs_type = os.getenv('ROOTFS_TYPE', 'developer')
    rootfs = rootfs_type.split()

    top_dir = os.environ.get('WORKSPACE', '.')
    input_dir = os.getenv('INPUT_DIR', 'artifacts')
    input_dir = os.path.join(top_dir, input_dir)
    output_dir = os.getenv('OUTPUT_DIR', 'out')
    output_dir = os.path.join(top_dir,
        output_dir,
        distribution,
        'pre-built',
        hwpack_type)

    lmc_cmd = 'linaro-media-create'
    lit = os.getenv('LIT')
    if lit is not None:
        arguments = ['rm', '-rf', 'lit']
        cmd_runner.run(arguments).wait()
        arguments = ['git', 'clone', lit, 'lit']
        cmd_runner.run(arguments).wait()
        lit_dir = os.path.join(top_dir, 'lit')
        lmc_cmd = os.path.join(lit_dir, lmc_cmd)

    for rootfs_type in rootfs:
        print 'INFO: Get hwpack/rootfs list'
        hwpack_file_name = 'hwpack_linaro-%s_' % hwpack_type
        rootfs_file_name = 'linaro-vivid-%s-' % rootfs_type
        for root, dirs, files in os.walk(input_dir):
            for file in files:
                if file.startswith(hwpack_file_name):
                    hwpack_file_name = os.path.join(root, file)
                    hwpack_file_basename = file
                if file.startswith(rootfs_file_name):
                    rootfs_file_name = os.path.join(root, file)
                    rootfs_file_basename = file

        print 'INFO: hwpack/rootfs found\nINFO: \t%s\nINFO: \t%s' % (
            hwpack_file_basename, rootfs_file_basename)

        device = HWPACK_DEV[hwpack_type]

        (hwpack_build_timestamp, hwpack_build_number) =\
        hwpack_file_basename.split('_')[2].split('-')

        distribution = rootfs_file_basename.split('-')[1]
        ret_split = rootfs_file_basename.split('.')[1].split('-')
        # rootfs_build_timestamp = ret_split[len(ret_split) - 2]
        rootfs_build_number = ret_split[len(ret_split) - 1]

        image_file = '%s-%s_%s_%s-%s.img' % (hwpack_type,
                                             distribution,
                                             rootfs_type,
                                             hwpack_build_timestamp,
                                             hwpack_build_number)
        image_size = ROOTFS_SIZE[rootfs_type]
        arguments = [lmc_cmd,
                     '--dev', device.lmc_name,
                     '--output-directory', os.path.join(output_dir,
                hwpack_build_number),
                     '--image-file', image_file,
                     '--image-size', image_size,
                     '--binary', rootfs_file_name,
                     '--hwpack', hwpack_file_name,
                     '--hwpack-force-yes'
        ]
        bootloader = os.environ.get("BOOTLOADER")
        if bootloader is None:
            print 'INFO: Use default bootloader (u_boot)'
        else:
            print 'INFO: Use specified bootlooader (%s)' % bootloader
            arguments.append('--bootloader')
            arguments.append(bootloader)
        print 'INFO: Create pre-built image(s)'
        before = time.time()
        env = os.environ
        env['LC_ALL'] = 'C'
        if lit is not None:
            env['PYTHONPATH'] = lit_dir
        proc = cmd_runner.Popen(arguments,
            env=env, stdout=subprocess.PIPE,
            stderr=subprocess.PIPE)
        (stdoutdata, stderrdata) = proc.communicate()
        if proc.returncode:
            print stdoutdata
            raise ValueError('linaro-media-create failed!\n%s' %
                             stderrdata)
        delta_min, delta_secs = divmod(int(time.time() - before), 60)
        print 'INFO: linaro-media-create - %02d:%02d' % (delta_min, delta_secs)

        image_file = os.path.join(output_dir, hwpack_build_number, image_file)
        arguments = ['gzip', '-9', image_file]
        print 'INFO: Compress pre-built image(s)'
        before = time.time()
        cmd_runner.run(arguments).wait()
        delta_min, delta_secs = divmod(int(time.time() - before), 60)
        print 'INFO: gzip - %02d:%02d' % (delta_min, delta_secs)

        image_file_url = '%s/%s/pre-built/%s/%s/%s.gz' %\
                         (snapshots_url,
                          distribution,
                          hwpack_type,
                          hwpack_build_number,
                          os.path.basename(image_file))
        arguments = ['zsyncmake',
                     '-b', '2048',
                     '-u', image_file_url,
                     '-o', '%s.gz.zsync' % image_file,
                     '%s.gz' % image_file
        ]
        print 'INFO: Create metafile(s) for zsync'
        before = time.time()
        cmd_runner.run(arguments).wait()
        delta_min, delta_secs = divmod(int(time.time() - before), 60)
        print 'INFO: zsyncmake - %02d:%02d' % (delta_min, delta_secs)

        print 'INFO: Create pre-built image information'
        md5 = hashlib.md5(open('%s.gz' % image_file).read()).hexdigest()

        template_file = os.path.join(top_dir, 'template.html')
        with open(template_file) as f:
            buffer = f.read()
            template = string.Template(buffer)

        hwpack_url = '%s/%s/%s/%s/%s/%s' %\
                     (snapshots_url,
                      distribution,
                      'hwpacks',
                      hwpack_type,
                      hwpack_build_number,
                      hwpack_file_basename)
        hwpack_manifest_url = ('%s' % hwpack_url).replace('.tar.gz',
            '.manifest.txt')
        hwpack_manifest = os.path.basename(hwpack_manifest_url)

        rootfs_url = '%s/%s/%s/%s/%s/%s' %\
                     (snapshots_url,
                      distribution,
                      'images',
                      rootfs_type,
                      rootfs_build_number,
                      rootfs_file_basename)
        rootfs_packages_url = ('%s' % rootfs_url).replace('.tar.gz',
            '.packages')
        rootfs_packages = os.path.basename(rootfs_packages_url)

        if distribution == 'openembedded':
            rootfs_packages_url = 'n/a'
            rootfs_packages = 'n/a'

        buffer = template.substitute(title='Pre-built image information',
            md5=md5, image_name=os.path.basename(image_file_url),
            hwpack_name=hwpack_file_basename, hwpack_url=hwpack_url,
            rootfs_name=rootfs_file_basename, rootfs_url=rootfs_url,
            hwpack_manifest_name=hwpack_manifest,
            hwpack_manifest_url=hwpack_manifest_url,
            rootfs_packages_name=rootfs_packages,
            rootfs_packages_url=rootfs_packages_url,
            build_number=os.environ.get('BUILD_NUMBER') or 'Not Found',
            build_url=os.environ.get('BUILD_URL') or 'Not Found')

        information_file = image_file.replace('.img', '.html')
        with open(information_file, 'w') as f:
            f.write(buffer)

        print 'INFO: Create license information'
        device.do_eula(image_file)

if __name__ == '__main__':
    main()