jenkins: Add build jobs

Adds two Jenkins pipeline jobs pb-upstream-trigger and pb-build-matrix.
pb-upstream-trigger checks for upstream updates and runs
pb-build-matrix.  pb-build-matrix builds a pb-builder image and runs the
build-pb script.

Signed-off-by: Geoff Levand <geoff@infradead.org>
---
Hi Sam,

Here is a first cut at Jenkins pipeline support.  The Jenkins
master will need the plugins: build-timeout, copyartifact, git,
pipeline, ssh-agent, workflow-aggregator, and maybe some others.

The matrix job expects worker nodes to have labels to indicate
the machine arch; amd64, arm64, etc., and the 'docker' tag to
indicate that the Jenkins account can run docker commands.

Now I have the default values for BUILD_ARCH_LIST and GIT_URL
hard coded into the jobs, but I think having a site specific
profile file that gets loaded would be a better solution as not
all sites will have say an arm64 Jenkins worker node.  Maybe
you would want to put something like 'ppc64le' in yours...

Please consider.

-Geoff

 jenkins/pb-build-matrix.groovy     | 81 ++++++++++++++++++++++++++++++++++++++
 jenkins/pb-upstream-trigger.groovy | 24 +++++++++++
 2 files changed, 105 insertions(+)
 create mode 100644 jenkins/pb-build-matrix.groovy
 create mode 100644 jenkins/pb-upstream-trigger.groovy

On Wed, 2018-05-23 at 17:25 -0700, Geoff Levand wrote:
> Adds two Jenkins pipeline jobs pb-upstream-trigger and pb-build-matrix.
> pb-upstream-trigger checks for upstream updates and runs
> pb-build-matrix.  pb-build-matrix builds a pb-builder image and runs the
> build-pb script.
> 
> Signed-off-by: Geoff Levand <geoff@infradead.org>

Thanks, merged as a30e4ac.
I have some changes in mind such as passing through proxy information and
configure options but I'll submit them once I've ironed out my own
Jenkins issues :)

> ---
> Hi Sam,
> 
> Here is a first cut at Jenkins pipeline support.  The Jenkins
> master will need the plugins: build-timeout, copyartifact, git,
> pipeline, ssh-agent, workflow-aggregator, and maybe some others.
> 
> The matrix job expects worker nodes to have labels to indicate
> the machine arch; amd64, arm64, etc., and the 'docker' tag to
> indicate that the Jenkins account can run docker commands.
> 
> Now I have the default values for BUILD_ARCH_LIST and GIT_URL
> hard coded into the jobs, but I think having a site specific
> profile file that gets loaded would be a better solution as not
> all sites will have say an arm64 Jenkins worker node.  Maybe
> you would want to put something like 'ppc64le' in yours...
> 
> Please consider.
> 
> -Geoff
> 
>  jenkins/pb-build-matrix.groovy     | 81 ++++++++++++++++++++++++++++++++++++++
>  jenkins/pb-upstream-trigger.groovy | 24 +++++++++++
>  2 files changed, 105 insertions(+)
>  create mode 100644 jenkins/pb-build-matrix.groovy
>  create mode 100644 jenkins/pb-upstream-trigger.groovy
> 
> diff --git a/jenkins/pb-build-matrix.groovy b/jenkins/pb-build-matrix.groovy
> new file mode 100644
> index 0000000..7547612
> --- /dev/null
> +++ b/jenkins/pb-build-matrix.groovy
> @@ -0,0 +1,81 @@
> +#!groovy
> +// Builds pb-builder image and runs build-pb script.
> +//
> +// The `jenkins` user must be in the `docker` user group.
> +// Requires nodes with labels: `amd64`, `arm64`, `docker`.
> +// Required plugins: build-timeout, copyartifact, git, pipeline, ssh-agent,
> +// workflow-aggregator.
> +
> +properties([
> +    buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '5')),
> +    parameters([
> +    string(name: 'BUILD_ARCH_LIST',
> +        defaultValue: 'amd64 arm64',
> +        description: 'List of Jenkins node architectures to build on.'),
> +    booleanParam(name: 'DOCKER_PURGE',
> +        defaultValue: false,
> +        description: 'Remove existing pb-builder docker image and rebuild.'),
> +    booleanParam(name: 'DRY_RUN',
> +        defaultValue: false,
> +        description: 'Dry run, do not build.'),
> +    string(name: 'GIT_URL',
> +        defaultValue: 'git://ozlabs.org/petitboot',
> +        description: 'URL of petitboot git repository.'),
> +    ])
> +])
> +
> +def build_pb = { String _build_arch, Boolean _dry_run, String _git_url,
> +    Boolean _purge
> +    ->
> +    String build_arch = _build_arch
> +    Boolean dry_run = _dry_run
> +    String git_url = _git_url
> +    Boolean purge = _purge
> +    String builder_args = ""
> +    String pb_args = ""
> +
> +    if (dry_run) {
> +        builder_args += " --dry-run"
> +        pb_args += " --dry-run"
> +    }
> +    if (purge) {
> +        builder_args += " --purge"
> +    }
> +
> +    // timeout if no build_arch node is available.
> +    timeout(time: 15, unit: 'MINUTES') {
> +        node("${build_arch} && docker") {
> +            git(poll: false, changelog: false, url: git_url)
> +
> +            stage("[${build_arch}--build-builder]") {
> +                sh("""./docker/build-builder --verbose ${builder_args}""")
> +            }
> +            stage("[${build_arch}--build-pb]") {
> +                sh("""./docker/build-pb --verbose --check ${pb_args}""")
> +            }
> +            stage('Post-build') {
> +                String result_file = "${BUILD_TAG}-${build_arch}-test-results.tar.xz"
> +                String test_info = """build_arch=${build_arch}
> +    BUILD_URL=${BUILD_URL}
> +    BUILD_TAG=${BUILD_TAG}
> +    GIT_URL=${GIT_URL}
> +    """
> +
> +                writeFile(file: 'test-info.txt', text: test_info)
> +                sh("tar -cJf ${result_file} test-info.txt test-suite.log \
> +                    \$(find test -name '*.log')")
> +                archiveArtifacts  "${result_file}"
> +            }
> +        }
> +    }
> +}
> +
> +def build_map = [:]
> +build_map.failFast = false
> +
> +for (build_arch in params.BUILD_ARCH_LIST.split()) {
> +    build_map[build_arch] = build_pb.curry(build_arch, params.DRY_RUN,
> +        params.GIT_URL, params.DOCKER_PURGE)
> +}
> +
> +parallel build_map
> diff --git a/jenkins/pb-upstream-trigger.groovy b/jenkins/pb-upstream-trigger.groovy
> new file mode 100644
> index 0000000..ab67e74
> --- /dev/null
> +++ b/jenkins/pb-upstream-trigger.groovy
> @@ -0,0 +1,24 @@
> +#!groovy
> +// Check for upstream updates and run builds.
> +
> +properties([
> +    buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '5')),
> +    pipelineTriggers([pollSCM('H/30 * * * *')]),
> +    parameters([
> +    string(name: 'GIT_URL',
> +        defaultValue: 'git://ozlabs.org/petitboot',
> +        description: 'URL of petitboot git repository.'),
> +    ])
> +])
> +
> +stage('Build') {
> +    node {
> +        git(poll: true, changelog: false, url: params.GIT_URL)
> +        build(
> +            job: 'pb-build-matrix',
> +            parameters: [
> +                string(name: 'GIT_URL', value: params.GIT_URL),
> +            ],
> +        )
> +    }
> +}