path: root/jenkins/bisect.jpl

#!/usr/bin/env groovy

/*
  Copyright (C) 2017 Collabora Limited
  Author: Guillaume Tucker <guillaume.tucker@collabora.com>

  This module is free software; you can redistribute it and/or modify it under
  the terms of the GNU Lesser General Public License as published by the Free
  Software Foundation; either version 2.1 of the License, or (at your option)
  any later version.

  This library is distributed in the hope that it will be useful, but WITHOUT
  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
  details.

  You should have received a copy of the GNU Lesser General Public License
  along with this library; if not, write to the Free Software Foundation, Inc.,
  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/

/* ----------------------------------------------------------------------------
 * Jenkins parameters

The following parameters need to be defined in the Jenkins pipeline job, with
typical default values in brackets:

KERNEL_URL
  URL of the kernel Git repository
KERNEL_BRANCH
  Name of the branch to bisect in the kernel Git repository
KERNEL_TREE
  Name of the kernel Git repository (tree)
GOOD_COMMIT
  Good known Git revision (SHA1 or any valid reference)
BAD_COMMIT
  Bad known Git revision (SHA1 or any valid reference)
ARCH
  CPU architecture as understood by the Linux kernel build system
DEFCONFIG (defconfig)
  Name of the Linux kernel defconfig
TARGET
  Name of the device type to test (typically LAVA device type name)
LAB
  Name of the lab in which to run the bisection tests
PLAN (boot)
  Name of the test plan
KCI_API_URL (https://api.kernelci.org)
  URL of the KernelCI backend API
KCI_TOKEN_ID
  Identifier of the KernelCI backend API token stored in Jenkins
KCI_STORAGE_URL (https://storage.kernelci.org/)
  URL of the KernelCI storage server
KCI_BUILD_URL (https://github.com/kernelci/kernelci-build.git)
  URL of the kernelci-build repository
KCI_BUILD_BRANCH (master)
  Name of the branch to use in the kernelci-build repository
LAVA_CI_URL (https://github.com/kernelci/lava-ci.git)
  URL of the lava-ci repository
LAVA_CI_BRANCH (master)
  Name of the branch to use in the lava-ci repository
LAVA_CALLBACK (kernel-ci-callback)
  Description of the LAVA auth token to look up and use in LAVA callbacks
LAVA_PRIORITY (low)
  Priority of all the LAVA tests
DISABLE_CONFIGS
  List of kernel config options to disable, separated by a whitespace
EMAIL_RECIPIENTS
  List of recipients for all emails generated by this job

 */


/* Working around some seemingly broken Python set-up... */
def eggCache() {
    def egg_cache = env.WORKSPACE + "/python-egg-cache"
    sh(script: "mkdir -p ${egg_cache}")
    return egg_cache
}

/* ----------------------------------------------------------------------------
 * git utilities
 */

def getSHA(kdir) {
    def sha

    dir(kdir) {
        sha = sh(script: "git rev-parse HEAD", returnStdout: true)
    }

    return sha
}

def gitDescribe(kdir) {
    def describe

    dir(kdir) {
        describe = sh(script: "git describe", returnStdout: true).trim()
    }

    return describe
}

def createTag(kdir, iteration) {
    def tag = gitDescribe(kdir)

    dir(kdir) {
        tag += "-${env.JOB_NAME}-${currentBuild.number}-${iteration}"
        sh(script: "git tag -a ${tag} -m ${tag} HEAD")
    }

    return tag
}

def removeTag(kdir, tag) {
    dir(kdir) {
        sh(script: "git tag -d ${tag}")
    }
}

def checkoutRevision(git_dir, git_rev) {
    dir(git_dir) {
        sh(script: """
git clean -fd && \
git checkout --detach ${git_rev}
""")
    }
}

/* ----------------------------------------------------------------------------
 * cloning projects
 */

def cloneKCIBuild(kci_build) {
    sh(script: "rm -rf ${kci_build}")
    dir("${kci_build}") {
        git(url: params.KCI_BUILD_URL,
            branch: params.KCI_BUILD_BRANCH,
            poll: false)
    }
}

def cloneLinux(kdir) {
    echo """
Initialising kernel tree
  url:    ${params.KERNEL_URL}
  branch: ${params.KERNEL_BRANCH}
  path:   ${kdir}"""
    sh """
if [ -d ${kdir}/.git ]; then
    cd ${kdir}
    if git remote | grep -e '^${params.KERNEL_TREE}\$'; then
        git remote set-url ${params.KERNEL_TREE} ${params.KERNEL_URL}
        git remote update ${params.KERNEL_TREE}
        git remote prune ${params.KERNEL_TREE}
    else
        git remote add ${params.KERNEL_TREE} ${params.KERNEL_URL}
        git remote update ${params.KERNEL_TREE}
    fi
else
    git clone ${params.KERNEL_URL} ${kdir} \
      -b ${params.KERNEL_BRANCH} \
      -o ${params.KERNEL_TREE}
    cd ${kdir}
fi
git config user.name 'kernelci.org bot'
git config user.email bot@kernelci.org
git reset --hard
echo 'build-*' > .git/info/exclude
git clean -fd
git bisect reset || echo -n
git checkout --detach HEAD || echo -n
git branch -D ${params.KERNEL_BRANCH} || echo -n
for t in \$(git tag -l | grep ${env.JOB_NAME}); do git tag -d \$t; done
git fetch ${params.KERNEL_TREE} ${params.KERNEL_BRANCH} --tags
git checkout FETCH_HEAD -b ${params.KERNEL_BRANCH}
git symbolic-ref HEAD refs/heads/${params.KERNEL_BRANCH}
cd -
"""
}

def cloneLAVA_CI(lava_ci) {
    sh(script: "rm -rf ${lava_ci}")
    dir("${lava_ci}") {
        git(url: params.LAVA_CI_URL,
            branch: params.LAVA_CI_BRANCH,
            poll: false)
    }
}

/* ----------------------------------------------------------------------------
 * kernel build
 */

def buildKernel(kdir, kci_build) {
    dir(kdir) {
        sh(script: "rm -f ${env._BUILD_JSON}")
        withCredentials([string(credentialsId: params.KCI_TOKEN_ID,
                                variable: 'SECRET')]) {
            sh(script: """
API=${params.KCI_API_URL} \
TOKEN=${SECRET} \
TREE_NAME=${params.KERNEL_TREE} \
TREE=${params.KERNEL_URL} \
ARCH=${params.ARCH} \
BRANCH=${params.KERNEL_BRANCH} \
""" + kci_build + """/build.py -e -g -i \
-j ${env._BUILD_JSON} \
-c ${params.DEFCONFIG}""")
        }
        stash(name: env._BUILD_JSON, includes: env._BUILD_JSON)
    }
}

def buildRevision(kdir, kci_build, git_rev, name) {
    checkoutRevision(kdir, git_rev)
    def tag = createTag(kdir, name)
    buildKernel(kdir, kci_build)
    return tag
}

/* ----------------------------------------------------------------------------
 * kernel test with LAVA v2
 */

def submitJob(lava_ci, describe, hook) {
    dir(lava_ci) {
        sh(script: "rm -rf ${env._BUILD_JSON}; rm -rf data; mkdir data")
        unstash(env._BUILD_JSON)

        withCredentials([string(credentialsId: params.KCI_TOKEN_ID,
                                variable: 'SECRET')]) {
            sh(script: """
./lava-v2-jobs-from-api.py \
--lab=${params.LAB} \
--builds=${env._BUILD_JSON} \
--storage=${params.KCI_STORAGE_URL} \
--plans=${params.PLAN} \
--jobs=data \
--arch=${params.ARCH} \
--tree=${params.KERNEL_TREE} \
--describe=${describe} \
--branch=${params.KERNEL_BRANCH} \
--defconfig_full=${params.DEFCONFIG} \
--priority=${params.LAVA_PRIORITY} \
--callback=${params.LAVA_CALLBACK} \
--callback-url=${hook.getURL()} \
--callback-dataset=results \
--callback-type=custom \
--targets=${params.TARGET}
""")
        }

        def egg_cache = eggCache()
        def token = "LAVA XMLRPC Token for " + params.LAB.split('-dev')[0]

        withCredentials([string(credentialsId: token, variable: 'SECRET')]) {
            sh(script: """
PYTHON_EGG_CACHE=${egg_cache} \
./lava-v2-submit-jobs.py \
--username=kernel-ci \
--token=${SECRET} \
--lab=${params.LAB} \
--jobs=data
""")
        }
    }
}

def getResult(lava_ci, hook) {
    def status = null

    dir(lava_ci) {
        echo "Waiting for job results..."
        def data = waitForWebhook(hook)
        def json_file = 'callback.json'
        writeFile(file: json_file, text: data)
        def token = "${params.LAB}-bisection-webhook"
        withCredentials([string(credentialsId: token, variable: 'SECRET')]) {
            def egg_cache = eggCache()
            status = sh(returnStatus: true, script: """
PYTHON_EGG_CACHE=${egg_cache} \
./lava-v2-callback.py \
--token=${SECRET} \
${json_file}
""")
        }
        sh(script: "rm -f ${json_file}")
    }

    return status
}

def runTest(lava_ci, describe) {
    cloneLAVA_CI(lava_ci)
    def hook = registerWebhook()
    submitJob(lava_ci, describe, hook)
    return getResult(lava_ci, hook)
}

/* ----------------------------------------------------------------------------
 * bisection
 */

def bisectStart(kdir) {
    def status = null

    dir(kdir) {
        status = sh(returnStatus: true, script: """
git bisect start
git bisect good ${params.GOOD_COMMIT}
git bisect bad ${params.BAD_COMMIT}
""")
    }

    return (status == 0) ? true : false
}

def bisectNext(kdir, status) {
    dir(kdir) {
        sh(script: "git clean -fd")

        switch (status) {
        case 0:
            sh(script: "git bisect good")
            break
        case 2:
            sh(script: "git bisect bad")
            break
        case 1:
            echo "Iteration failed, skipping"
            sh(script: "git bisect skip")
            break
        default:
            echo "Unexpected status, skipping"
            sh(script: "git bisect skip")
            break
        }
    }
}

/* ----------------------------------------------------------------------------
 * pipeline
 */

def runCheck(kdir, kci_build, git_commit, name, run_status) {
    def check = null
    def tag = null

    lock("${env.NODE_NAME}-build-lock") {
        timeout(time: 60, unit: 'MINUTES') {
            try {
                tag = buildRevision(kdir, kci_build, git_commit, name)
                check = true
            } catch (error) {
                check = false
            }
        }
    }

    if (!check)
        return false

    def describe = gitDescribe(kdir)

    node("kernel-boot-v2") {
        timeout(time: 120, unit: 'MINUTES') {
            def lava_ci = env.WORKSPACE + '/lava-ci'
            def status = runTest(lava_ci, describe)
            check = (status == run_status ? true : false)
        }
    }

    removeTag(kdir, tag)

    return check
}

def checkAbort(passed, message) {
    if (!passed) {
        echo message
        currentBuild.result = 'ABORTED'
    }

    return passed
}

node("bisection") {
    /* Global pipeline constants */
    env._BUILD_JSON = "build-data.json"

    def kci_build = env.WORKSPACE + '/kernelci-build'
    def kdir = env.WORKSPACE + '/linux'
    def check = null
    def checks = [:]

    def params_summary = """\
    Tree:      ${params.KERNEL_TREE}
    URL:       ${params.KERNEL_URL}
    Branch:    ${params.KERNEL_BRANCH}
    Target:    ${params.TARGET}
    Lab:       ${params.LAB}
    Defconfig: ${params.DEFCONFIG}
    Plan:      ${params.PLAN}"""
    echo """\
    Good:      ${params.GOOD_COMMIT}
    Bad:       ${params.BAD_COMMIT}
${params_summary}"""

    if ((params.PLAN != 'boot') && (params.PLAN != 'simple')) {
        echo "Only doing boot and simple plans for now, aborting."
        currentBuild.result = 'ABORTED'
        return
    }

    try {
        stage("Init") {
            timeout(time: 30, unit: 'MINUTES') {
                parallel(
                    p1: { cloneKCIBuild(kci_build) },
                    p2: { cloneLinux(kdir) },
                )
            }
        }

        stage("Check pass") {
            check = runCheck(kdir, kci_build, params.GOOD_COMMIT, 'pass', 0)
        }
        if (!checkAbort(check, "Good revision check failed"))
            return

        stage("Check fail") {
            check = runCheck(kdir, kci_build, params.BAD_COMMIT, 'fail', 2)
        }
        if (!checkAbort(check, "Bad revision check failed"))
            return

        stage("Start") {
            timeout(time: 5, unit: 'MINUTES') {
                check = bisectStart(kdir)
            }
        }
        if (!checkAbort(check,
                   "Failed to start bisection, commits range may be invalid."))
            return

        def previous = params.GOOD_COMMIT
        def current = getSHA(kdir)
        def iteration = 1

        while (previous != current) {
            def tag = createTag(kdir, iteration)
            def status = null

            echo "Iteration #${iteration}: ${tag}"

            lock("${env.NODE_NAME}-build-lock") {
                stage("Build ${iteration}") {
                    timeout(time: 60, unit: 'MINUTES') {
                        try {
                            buildKernel(kdir, kci_build)
                            status = 0
                        } catch (error) {
                            status = 1
                        }
                    }
                }
            }

            if (status == 0) {
                def describe = gitDescribe(kdir)

                node("kernel-boot-v2") {
                    stage("Test ${iteration}") {
                        timeout(time: 120, unit: 'MINUTES') {
                            def lava_ci = env.WORKSPACE + '/lava-ci'
                            status = runTest(lava_ci, describe)
                        }
                    }
                }
            }

            removeTag(kdir, tag)

            stage("Next") {
                timeout(time: 5, unit: 'MINUTES') {
                    bisectNext(kdir, status)
                }
            }

            previous = current
            current = getSHA(kdir)
            iteration += 1
        }

        stage("Verify") {
            check = runCheck(kdir, kci_build, 'refs/bisect/bad', 'verify', 2)
            checks['verify'] = check ? 'PASS' : 'FAIL'
        }
        if (!checkAbort(check, "Result check failed"))
            return

        stage("Revert") {
            dir(kdir) {
                sh(script: "git revert refs/bisect/bad")
            }
            check = runCheck(kdir, kci_build, 'HEAD', 'revert', 0)
            checks['revert'] = check ? 'PASS' : 'FAIL'
        }
        if (!check)
            echo "Warning: revert check failed"

    } catch (err) {
        currentBuild.result = "FAILURE"

        def tree_branch = "${params.KERNEL_TREE}/${params.KERNEL_BRANCH}"
        def subject = "bisection error: #${env.BUILD_NUMBER} \
${tree_branch} ${params.LAB} ${params.TARGET}"
        def body = """\
${env.BUILD_URL}

${params_summary}

${err}
"""
        emailext(subject: subject, body: body, to: params.EMAIL_RECIPIENTS)

        throw err
    }
}