path: root/jenkins/bisect.jpl

#!/usr/bin/env groovy

/*
  Copyright (C) 2017, 2018 Collabora Limited
  Author: Guillaume Tucker <guillaume.tucker@collabora.com>

  This module is free software; you can redistribute it and/or modify it under
  the terms of the GNU Lesser General Public License as published by the Free
  Software Foundation; either version 2.1 of the License, or (at your option)
  any later version.

  This library is distributed in the hope that it will be useful, but WITHOUT
  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
  details.

  You should have received a copy of the GNU Lesser General Public License
  along with this library; if not, write to the Free Software Foundation, Inc.,
  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/

/* ----------------------------------------------------------------------------
 * Jenkins parameters

The following parameters need to be defined in the Jenkins pipeline job, with
typical default values in brackets:

KERNEL_URL
  URL of the kernel Git repository
KERNEL_BRANCH
  Name of the branch to bisect in the kernel Git repository
KERNEL_TREE
  Name of the kernel Git repository (tree)
KERNEL_NAME
  Identifier of the kernel (typically `git describe`)
GOOD_COMMIT
  Good known Git revision (SHA1 or any valid reference)
BAD_COMMIT
  Bad known Git revision (SHA1 or any valid reference)
REF_KERNEL_URL (git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git)
  URL of the reference kernel Git repository used to find merge bases
REF_KERNEL_BRANCH (master)
  Name of the branch from the reference kernel Git repository
REF_KERNEL_TREE (mainline)
  Name of the reference kernel Git repository
ARCH
  CPU architecture as understood by the Linux kernel build system
DEFCONFIG (defconfig)
  Name of the Linux kernel defconfig
TARGET
  Name of the device type to test (typically LAVA device type name)
BUILD_ENVIRONMENT
  Name of the build environment
LAB
  Name of the lab in which to run the bisection tests
PLAN (boot)
  Name of the test plan
TEST_RUNS (1)
  Number of LAVA jobs to run before considering pass or fail.
KCI_API_URL (https://api.kernelci.org)
  URL of the KernelCI backend API
KCI_TOKEN_ID
  Identifier of the KernelCI backend API token stored in Jenkins
KCI_STORAGE_URL (https://storage.kernelci.org/)
  URL of the KernelCI storage server
KCI_CORE_URL (https://github.com/kernelci/kernelci-core.git)
  URL of the kernelci-core repository
KCI_CORE_BRANCH (master)
  Name of the branch to use in the kernelci-core repository
DOCKER_BASE (kernelci/build-)
  Dockerhub base address used for the build images
LAVA_CALLBACK (kernel-ci-callback)
  Description of the LAVA auth token to look up and use in LAVA callbacks
LAVA_PRIORITY (medium)
  Priority of all the LAVA tests
EMAIL_RECIPIENTS
  List of recipients for all emails generated by this job
LABS_WHITELIST
  If defined, jobs will abort if the LAB is not on that list.
TREES_WHITELIST
  If defined, jobs will abort if the KERNEL_TREE is not on that list.

 */


@Library('kernelci') _
import org.kernelci.build.Kernel
import org.kernelci.util.Job

/* Working around some seemingly broken Python set-up... */
def eggCache() {
    def egg_cache = env.WORKSPACE + "/python-egg-cache"
    sh(script: "mkdir -p ${egg_cache}")
    return egg_cache
}

/* ----------------------------------------------------------------------------
 * git utilities
 */

def getSHA(kdir) {
    def sha

    dir(kdir) {
        sha = sh(script: "git rev-parse HEAD", returnStdout: true)
    }

    return sha
}

def gitDescribe(kdir) {
    def describe

    dir(kdir) {
        describe = sh(script: "git describe", returnStdout: true).trim()
    }

    return describe
}

def createTag(kdir, iteration) {
    def tag = gitDescribe(kdir)

    dir(kdir) {
        tag += "-${env.JOB_NAME}-${currentBuild.number}-${iteration}"
        sh(script: "git tag -a ${tag} -m ${tag} HEAD")
    }

    return tag
}

def removeTag(kdir, tag) {
    dir(kdir) {
        sh(script: "git tag -d ${tag}")
    }
}

def checkoutRevision(git_dir, git_rev) {
    dir(git_dir) {
        sh(script: """
git clean -fd && \
git checkout --detach ${git_rev}
""")
    }
}

def setRemote(kdir, name, url) {
    dir(kdir) {
        sh(script: """
if git remote | grep -e '^${name}\$'; then
    git remote set-url ${name} ${url}
    git remote update ${name}
    git remote prune ${name}
else
    git remote add ${name} ${url}
    git remote update ${name}
fi
""")
    }
}

/* ----------------------------------------------------------------------------
 * cloning projects
 */

def cloneKciCore(kci_core) {
    def j = new Job()
    j.cloneKciCore(kci_core, params.KCI_CORE_URL, params.KCI_CORE_BRANCH)
}

def cloneLinux(kdir) {
    print("""\
Initialising kernel tree
  url:    ${params.KERNEL_URL}
  branch: ${params.KERNEL_BRANCH}
  path:   ${kdir}""")

    if (sh(returnStatus: true, script: "test -d ${kdir}/.git") == 0) {
        setRemote(kdir, params.KERNEL_TREE, params.KERNEL_URL)
    } else {
        sh(script: """
git clone ${params.KERNEL_URL} ${kdir} \
  -b ${params.KERNEL_BRANCH} \
  -o ${params.KERNEL_TREE}
""")
    }

    dir(kdir) {
        sh(script: """
git config user.name 'kernelci.org bot'
git config user.email bot@kernelci.org
git reset --hard
echo 'build-*' > .git/info/exclude
git clean -fd
git bisect reset || echo -n
git checkout --detach HEAD || echo -n
git branch -D ${params.KERNEL_BRANCH} || echo -n
for t in \$(git tag -l | grep ${env.JOB_NAME}); do git tag -d \$t; done
git fetch ${params.KERNEL_TREE} ${params.KERNEL_BRANCH} --tags -f
git checkout ${params.BAD_COMMIT} -b ${params.KERNEL_BRANCH}
git symbolic-ref HEAD refs/heads/${params.KERNEL_BRANCH}
""")
    }
}

/* ----------------------------------------------------------------------------
 * kernel build
 */

def buildKernel(kdir, kci_core) {
    def output = "${kdir}/build-${params.ARCH}-${params.BUILD_ENVIRONMENT}"
    dir(kci_core) {
        sh(script: "rm -f ${env._BUILD_JSON}")

        sh(script: """\
for d in \$(find ${kdir} -name "build-*" -type d); do
  [ "\$d" = "${output}" ] || rm -rf "\$d"
done
""")

        sh(script:"""\
./kci_build \
generate_defconfig_fragments \
--defconfig=${params.DEFCONFIG} \
--kdir=${kdir} \
""")

        def expanded_defconfig = sh(script: """\
./kci_build \
expand_fragments \
--defconfig=${params.DEFCONFIG} \
""", returnStdout: true).trim()

        sh(script:"""\
./kci_build \
build_kernel \
--kdir=${kdir} \
--defconfig=${expanded_defconfig} \
--arch=${params.ARCH} \
--build-env=${params.BUILD_ENVIRONMENT} \
--output=${output} \
""")

        sh(script: """\
./kci_build \
install_kernel \
--kdir=${kdir} \
--tree-name=${params.KERNEL_TREE} \
--tree-url=${params.KERNEL_URL} \
--branch=${params.KERNEL_BRANCH} \
--output=${output} \
""")

        withCredentials([string(credentialsId: params.KCI_TOKEN_ID,
                                variable: 'SECRET')]) {
            sh(script: """\
./kci_build \
push_kernel \
--kdir=${kdir} \
--token=${SECRET} \
--api=${params.KCI_API_URL} \
""")
        }

        sh(script: """\
./kci_build \
publish_kernel \
--kdir=${kdir} \
--json-path=${env._BUILD_JSON} \
""")

        stash(name: env._BUILD_JSON, includes: env._BUILD_JSON)
    }
}

def buildRevision(kdir, kci_core, git_rev, name) {
    checkoutRevision(kdir, git_rev)
    def tag = createTag(kdir, name)
    buildKernel(kdir, kci_core)
    return tag
}

/* ----------------------------------------------------------------------------
 * kernel test with LAVA v2
 */

def submitJob(kci_core, describe, hook) {
    dir(kci_core) {
        sh(script: "rm -rf ${env._BUILD_JSON}; rm -rf data; mkdir data")
        unstash(env._BUILD_JSON)
        sh(script: """
./lava-v2-jobs-from-api.py \
--lab=${params.LAB} \
--builds=${env._BUILD_JSON} \
--storage=${params.KCI_STORAGE_URL} \
--plans=${params.PLAN} \
--jobs=data \
--arch=${params.ARCH} \
--tree=${params.KERNEL_TREE} \
--describe=${describe} \
--branch=${params.KERNEL_BRANCH} \
--defconfig_full=${params.DEFCONFIG} \
--priority=${params.LAVA_PRIORITY} \
--callback=${params.LAVA_CALLBACK} \
--callback-url=${hook.getURL()} \
--callback-dataset=results \
--callback-type=custom \
--targets=${params.TARGET}
""")

        def egg_cache = eggCache()
        def token = "${params.LAB}-lava-api"

        withCredentials([string(credentialsId: token, variable: 'SECRET')]) {
            sh(script: """
PYTHON_EGG_CACHE=${egg_cache} \
./lava-v2-submit-jobs.py \
--username=kernel-ci \
--token=${SECRET} \
--lab=${params.LAB} \
--jobs=data
""")
        }
    }
}

def getResult(kci_core, hook) {
    def status = null

    dir(kci_core) {
        echo "Waiting for job results..."
        def data = waitForWebhook(hook)
        def json_file = 'callback.json'
        writeFile(file: json_file, text: data)
        def token = "${params.LAB}-bisection-webhook"
        withCredentials([string(credentialsId: token, variable: 'SECRET')]) {
            def egg_cache = eggCache()
            status = sh(returnStatus: true, script: """
PYTHON_EGG_CACHE=${egg_cache} \
./lava-v2-callback.py \
--token=${SECRET} \
${json_file}
""")
        }
        sh(script: "rm -f ${json_file}")
    }

    return status
}

def runTest(kci_core, describe, expected=0, runs=0) {
    if (!runs)
        runs = params.TEST_RUNS.toInteger()

    def status = null

    for (int i = 1; i <= runs; i++) {
        echo "Run ${i} / ${runs}"

        def retries = 3

        while (retries) {
            def hook = registerWebhook()
            submitJob(kci_core, describe, hook)
            status = getResult(kci_core, hook)

            if (status == 1)
                retries -= 1
            else
                break
        }

        if (status != expected)
            break
    }

    return status
}

/* ----------------------------------------------------------------------------
 * bisection
 */

def findMergeBase(kdir, good, bad) {
    def base = good

    dir(kdir) {
        def good_base = sh(
            returnStatus: true,
            script: "git merge-base --is-ancestor ${base} HEAD")

        if (good_base != 0) {
            def ref = "${params.REF_KERNEL_TREE}/${params.REF_KERNEL_BRANCH}"
            print("Good commit not in current branch, finding base in ${ref}")

            print("""\
Reference:
    Tree:      ${params.REF_KERNEL_TREE}
    URL:       ${params.REF_KERNEL_URL}
    Branch:    ${params.REF_KERNEL_BRANCH}""")

            setRemote(kdir, params.REF_KERNEL_TREE, params.REF_KERNEL_URL)
            base = sh(script: "git merge-base ${bad} ${ref}",
                      returnStdout: true).trim()
            print("Merge base: ${base}")
        }
    }

    return base
}

def bisectStart(kdir, good, bad) {
    def status = null

    dir(kdir) {
        status = sh(returnStatus: true, script: """
git bisect start
git bisect good ${good}
git bisect bad ${bad}
""")
    }

    return (status == 0) ? true : false
}

def bisectNext(kdir, status) {
    dir(kdir) {
        sh(script: "git clean -fd")

        switch (status) {
        case 0:
            sh(script: "git bisect good")
            break
        case 2:
            sh(script: "git bisect bad")
            break
        case 1:
            echo "Iteration failed, skipping"
            sh(script: "git bisect skip")
            break
        default:
            echo "Unexpected status, skipping"
            sh(script: "git bisect skip")
            break
        }
    }
}

/* ----------------------------------------------------------------------------
 * Results
 */

def pushResults(kci_core, kdir, checks, params_summary) {
    def subject = "${params.KERNEL_TREE}/${params.KERNEL_BRANCH} ${params.PLAN} bisection: ${params.KERNEL_NAME} on ${params.TARGET}"

    dir(kci_core) {
        withCredentials([string(credentialsId: params.KCI_TOKEN_ID,
                                variable: 'SECRET')]) {
            def egg_cache = eggCache()
            sh(script: """
PYTHON_EGG_CACHE=${egg_cache} \
./push-bisection-results.py \
--token=${SECRET} \
--api=${params.KCI_API_URL} \
--lab=${params.LAB} \
--arch=${params.ARCH} \
--defconfig=${params.DEFCONFIG} \
--build-environment=${params.BUILD_ENVIRONMENT} \
--target=${params.TARGET} \
--tree=${params.KERNEL_TREE} \
--kernel=${params.KERNEL_NAME} \
--branch=${params.KERNEL_BRANCH} \
--good=${params.GOOD_COMMIT} \
--bad=${params.BAD_COMMIT} \
--verify=${checks['verify']} \
--revert=${checks['revert']} \
--kdir=${kdir} \
--subject=\"${subject}\" \
--to=\"${params.EMAIL_RECIPIENTS}\" \
""")
        }
    }
}

/* ----------------------------------------------------------------------------
 * pipeline
 */

def runCheck(kdir, kci_core, git_commit, name, run_status, runs=0) {
    def check = null
    def tag = null

    lock("${env.NODE_NAME}-build-lock") {
        timeout(time: 60, unit: 'MINUTES') {
            try {
                tag = buildRevision(kdir, kci_core, git_commit, name)
                check = true
            } catch (error) {
                check = false
            }
        }
    }

    if (!check)
        return false

    def describe = gitDescribe(kdir)

    timeout(time: 120, unit: 'MINUTES') {
        def status = runTest(kci_core, describe, run_status, runs)
        check = (status == run_status ? true : false)
    }

    removeTag(kdir, tag)

    return check
}

def checkAbort(passed, message) {
    if (!passed) {
        echo message
        currentBuild.result = 'ABORTED'
    }

    return passed
}

def bisection(kci_core, kdir, checks) {
    def check = null
    def good = null
    def bad = null

    stage("Init") {
        timeout(time: 30, unit: 'MINUTES') {
            parallel(
                kci_core: { cloneKciCore(kci_core) },
                kdir: { cloneLinux(kdir) },
            )
        }

        bad = params.BAD_COMMIT
        good = findMergeBase(kdir, params.GOOD_COMMIT, bad)
    }

    stage("Check pass") {
        check = runCheck(kdir, kci_core, good, 'pass', 0)
    }
    if (!checkAbort(check, "Good revision check failed"))
        return check

    stage("Check fail") {
        check = runCheck(kdir, kci_core, bad, 'fail', 2)
    }
    if (!checkAbort(check, "Bad revision check failed"))
        return check

    stage("Start") {
        timeout(time: 5, unit: 'MINUTES') {
            check = bisectStart(kdir, good, bad)
        }
    }
    if (!checkAbort(check,
                    "Failed to start bisection, commits range may be invalid."))
        return check

    def previous = good
    def current = getSHA(kdir)
    def iteration = 1

    while (previous != current) {
        def tag = createTag(kdir, iteration)
        def status = null

        echo "Iteration #${iteration}: ${tag}"

        lock("${env.NODE_NAME}-build-lock") {
            stage("Build ${iteration}") {
                timeout(time: 60, unit: 'MINUTES') {
                    try {
                        buildKernel(kdir, kci_core)
                        status = 0
                    } catch (error) {
                        status = 1
                    }
                }
            }
        }

        if (status == 0) {
            def describe = gitDescribe(kdir)

            stage("Test ${iteration}") {
                timeout(time: 120, unit: 'MINUTES') {
                    status = runTest(kci_core, describe)
                }
            }
        }

        removeTag(kdir, tag)

        stage("Next") {
            timeout(time: 5, unit: 'MINUTES') {
                bisectNext(kdir, status)
            }
        }

        previous = current
        current = getSHA(kdir)
        iteration += 1
    }

    stage("Verify") {
        check = runCheck(kdir, kci_core, 'refs/bisect/bad', 'verify', 2, 3)
        checks['verify'] = check ? 'PASS' : 'FAIL'
    }
    if (!checkAbort(check, "Result check failed"))
        return check

    stage("Revert") {
        dir(kdir) {
            sh(script: "git revert refs/bisect/bad")
        }
        check = runCheck(kdir, kci_core, 'HEAD', 'revert', 0, 3)
        checks['revert'] = check ? 'PASS' : 'FAIL'
    }
    if (!check)
        echo "Warning: revert check failed"

    return true
}

node("docker && bisection") {
    /* Global pipeline constants */
    env._BUILD_JSON = "build-data.json"

    def j = new Job()
    def kci_core = "${env.WORKSPACE}/kernelci-core"
    def kdir = "${env.WORKSPACE}/linux"
    def checks = [:]
    def docker_image = null

    def params_summary = """\
    Tree:      ${params.KERNEL_TREE}
    URL:       ${params.KERNEL_URL}
    Branch:    ${params.KERNEL_BRANCH}
    Kernel:    ${params.KERNEL_NAME}
    Target:    ${params.TARGET}
    Lab:       ${params.LAB}
    Defconfig: ${params.DEFCONFIG}
    Compiler:  ${params.BUILD_ENVIRONMENT}
    Plan:      ${params.PLAN}"""
    print("""\
    Good:      ${params.GOOD_COMMIT}
    Bad:       ${params.BAD_COMMIT}
${params_summary}""")

    if ((params.PLAN != 'boot') && (params.PLAN != 'simple')) {
        echo "Only doing boot and simple plans for now, aborting."
        currentBuild.result = 'ABORTED'
        return
    }

    if (params.LABS_WHITELIST) {
        def labs = params.LABS_WHITELIST.tokenize(' ')

        if (!labs.contains(params.LAB)) {
            echo "Lab not on whitelist, aborting."
            currentBuild.result = 'ABORTED'
            return
        }
    }

    if (params.TREES_WHITELIST) {
        def trees = params.TREES_WHITELIST.tokenize(' ')

        if (!trees.contains(params.KERNEL_TREE)) {
            echo "Tree not on whitelist, aborting."
            currentBuild.result = 'ABORTED'
            return
        }
    }

    j.dockerPullWithRetry("${params.DOCKER_BASE}base").inside() {
        j.cloneKciCore(kci_core, params.KCI_CORE_URL, params.KCI_CORE_BRANCH)
        build_env_docker_image = j.dockerImageName(
            kci_core, params.BUILD_ENVIRONMENT, params.ARCH)
        docker_image = "${params.DOCKER_BASE}${build_env_docker_image}"
    }

    j.dockerPullWithRetry(docker_image).inside() {
        try {
            def valid_bisect = bisection(kci_core, kdir, checks)
            if (!valid_bisect)
                return
        } catch (err) {
            currentBuild.result = "FAILURE"

            def tree_branch = "${params.KERNEL_TREE}/${params.KERNEL_BRANCH}"
            def subject = "bisection error: #${env.BUILD_NUMBER} \
${tree_branch} ${params.LAB} ${params.TARGET}"
            def body = """\
${env.BUILD_URL}

${params_summary}

${err}
"""
            emailext(subject: subject, body: body, to: params.EMAIL_RECIPIENTS)

            throw err
        }

        stage("Report") {
            pushResults(kci_core, kdir, checks, params_summary)
        }
    }
}