[PATCH v2 24/33] bootlinux: Add a new builder choice

[Luis Chamberlain <mcgrof@kernel.org>]

From: Chuck Lever <chuck.lever@oracle.com>

Currently, for cloud configurations or when 9p is disabled, kdevops
builds the test kernel on each test runner. This is inefficient,
and gets worse as the test matrix for a single kernel version scales
out.

Instead we want to build the test kernel once and make those build
artifacts available for test runners to install. This would be
similar to what KOTD does now, except it does not require setting up
a separate yum repo.

Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
---
 playbooks/roles/bootlinux/defaults/main.yml   |   2 +
 .../roles/bootlinux/tasks/build/builder.yml   | 228 ++++++++++++++++++
 .../tasks/install-deps/redhat/main.yml        |  18 ++
 playbooks/roles/bootlinux/tasks/main.yml      |   6 +
 4 files changed, 254 insertions(+)
 create mode 100644 playbooks/roles/bootlinux/tasks/build/builder.yml

diff --git a/playbooks/roles/bootlinux/defaults/main.yml b/playbooks/roles/bootlinux/defaults/main.yml
index 4a1ad72e..50ce96bf 100644
--- a/playbooks/roles/bootlinux/defaults/main.yml
+++ b/playbooks/roles/bootlinux/defaults/main.yml
@@ -46,3 +46,5 @@ kdevops_workflow_enable_cxl: False
 
 bootlinux_cxl_test: False
 bootlinux_tree_set_by_cli: False
+
+bootlinux_artifacts_dir: "{{ topdir_path }}/workflows/linux/artifacts"
diff --git a/playbooks/roles/bootlinux/tasks/build/builder.yml b/playbooks/roles/bootlinux/tasks/build/builder.yml
new file mode 100644
index 00000000..c8363b8e
--- /dev/null
+++ b/playbooks/roles/bootlinux/tasks/build/builder.yml
@@ -0,0 +1,228 @@
+---
+- name: Install b4
+  become: true
+  become_method: ansible.builtin.sudo
+  ansible.builtin.pip:
+    name:
+      - b4
+  when:
+    - target_linux_install_b4 is defined
+    - target_linux_install_b4
+    - ansible_os_family == "Debian"
+
+- name: Clone {{ target_linux_tree }}
+  ansible.builtin.git:
+    repo: "{{ target_linux_git }}"
+    dest: "{{ target_linux_dir_path }}"
+    update: true
+    depth: "{{ target_linux_shallow_depth }}"
+    version: "{{ target_linux_ref }}"
+  register: result
+  retries: 3
+  delay: 5
+  until: result is succeeded
+
+- name: Copy the kernel delta to the builder
+  ansible.builtin.template:
+    src: "{{ target_linux_extra_patch }}"
+    dest: "{{ target_linux_dir_path }}/{{ target_linux_extra_patch }}"
+    owner: "{{ data_user }}"
+    group: "{{ data_group }}"
+    mode: "u=rw,g=r,o=r"
+  when:
+    - target_linux_extra_patch is defined
+
+- name: Apply the kernel delta on the builder
+  # noqa: command-instead-of-module
+  ansible.builtin.command:
+    cmd: "git am {{ target_linux_extra_patch }}"
+    chdir: "{{ target_linux_dir_path }}"
+  register: git_am
+  changed_when: not git_am.failed
+  when:
+    - target_linux_extra_patch is defined
+
+- name: Check git user name and email configuration
+  when:
+    - target_linux_apply_patch_message_id is defined
+    - target_linux_apply_patch_message_id | length > 0
+    - bootlinux_b4_am_this_host|bool
+  block:
+    - name: Get the user's git config info
+      community.general.git_config_info:
+        scope: global
+      register: git_user_info
+
+    - name: Set dummy git user email address
+      community.general.git_config:
+        name: user.email
+        scope: global
+        value: "user@example.com"
+      when:
+        - '"user.email" not in git_user_info.config_values'
+
+    - name: Set dummy git user name
+      community.general.git_config:
+        name: user.name
+        scope: global
+        value: "Kdevops User"
+      when:
+        - '"user.name" not in git_user_info.config_values'
+
+- name: Show the message_id
+  ansible.builtin.debug:
+    msg: "{{ target_linux_apply_patch_message_id }}"
+  when:
+    - target_linux_apply_patch_message_id is defined
+
+- name: Apply a message patch set
+  ansible.builtin.shell:
+    chdir: "{{ target_linux_dir_path }}"
+    cmd: |
+      set -o pipefail
+      b4 am -o - {{ target_linux_apply_patch_message_id }} | git am
+  register: b4_am
+  changed_when: not b4_am.failed
+  when:
+    - target_linux_apply_patch_message_id is defined
+    - target_linux_apply_patch_message_id | length > 0
+    - bootlinux_b4_am_this_host|bool
+
+- name: Copy the configuration for kernel {{ target_linux_tree }}
+  ansible.builtin.template:
+    src: "{{ role_path }}/templates/{{ linux_config }}"
+    dest: "{{ target_linux_dir_path }}/.config"
+    owner: "{{ data_user }}"
+    group: "{{ data_group }}"
+    mode: "u=rw,g=r,o=r"
+
+- name: Set the kernel localversion
+  ansible.builtin.lineinfile:
+    path: "{{ target_linux_dir_path }}/localversion"
+    line: "{{ target_linux_localversion }}"
+    mode: "u=rw,g=r,o=r"
+    create: true
+  when:
+    - target_linux_localversion is defined
+    - target_linux_localversion != ""
+
+- name: Configure kernel {{ target_linux_tree }}
+  community.general.make:
+    chdir: "{{ target_linux_dir_path }}"
+    target: "olddefconfig"
+
+- name: Build {{ target_linux_tree }}
+  community.general.make:
+    chdir: "{{ target_linux_dir_path }}"
+    jobs: "{{ ansible_processor_nproc }}"
+    target: "all"
+
+- name: Remove the old artifacts directory on the control host
+  delegate_to: localhost
+  ansible.builtin.file:
+    path: "{{ bootlinux_artifacts_dir }}"
+    state: absent
+
+- name: Ensure an empty artifacts directory exists on the control host
+  delegate_to: localhost
+  run_once: true
+  ansible.builtin.file:
+    path: "{{ bootlinux_artifacts_dir }}"
+    state: directory
+    mode: "u=rwx,g=rx,o=rx"
+
+- name: Build kernel .deb packages
+  when:
+    - ansible_os_family == "Debian"
+  block:
+    - name: Make the bindeb-pkg target
+      community.general.make:
+        chdir: "{{ target_linux_dir_path }}"
+        jobs: "{{ ansible_processor_nproc }}"
+        target: "bindeb-pkg"
+
+    - name: Find the build artifacts
+      ansible.builtin.find:
+        paths: "{{ artifact_paths }}"
+        patterns: "*.deb"
+        file_type: file
+        recurse: true
+      register: found_debs
+
+    - name: Fetch the build artifacts to the control host
+      ansible.builtin.fetch:
+        src: "{{ item.path }}"
+        dest: "{{ bootlinux_artifacts_dir }}/"
+        flat: true
+      loop: "{{ found_debs.files }}"
+      loop_control:
+        label: "Fetching {{ item.path | basename }} ..."
+
+- name: Build kernel .rpm packages
+  when:
+    - ansible_os_family != "Debian"
+  block:
+    - name: Build the list of artifacts directories on the builder
+      ansible.builtin.set_fact:
+        artifact_paths:
+          - "{{ target_linux_dir_path }}/rpmbuild/RPMS"
+          - "{{ ansible_env.HOME }}/rpmbuild/RPMS"
+
+    - name: Wipe the artifact directories on the builder
+      ansible.builtin.file:
+        path: "{{ item }}"
+        state: absent
+      loop: "{{ artifact_paths }}"
+      loop_control:
+        label: "Deleting {{ item }} ..."
+
+    - name: Create empty artifact directories on the builder
+      ansible.builtin.file:
+        path: "{{ item }}"
+        state: directory
+        mode: "u=rwx,g=rx,o=rx"
+      loop: "{{ artifact_paths }}"
+      loop_control:
+        label: "Creating {{ item }} ..."
+
+    - name: Make the binrpm-pkg target
+      community.general.make:
+        chdir: "{{ target_linux_dir_path }}"
+        jobs: "{{ ansible_processor_nproc }}"
+        target: "binrpm-pkg"
+        params:
+          RPMOPTS: '--without devel'
+
+    - name: Find the build artifacts
+      ansible.builtin.find:
+        paths: "{{ artifact_paths }}"
+        patterns: "*.rpm"
+        file_type: file
+        recurse: true
+      register: found_rpms
+
+    - name: Fetch the build artifacts to the control host
+      ansible.builtin.fetch:
+        src: "{{ item.path }}"
+        dest: "{{ bootlinux_artifacts_dir }}/"
+        flat: true
+      loop: "{{ found_rpms.files }}"
+      loop_control:
+        label: "Fetching {{ item.path | basename }} ..."
+
+- name: Extract the release information of the built kernel
+  community.general.make:
+    chdir: "{{ target_linux_dir_path }}"
+    target: "kernelrelease"
+  register: kernelrelease
+
+- name: Store the kernel release information with the build artifacts
+  delegate_to: localhost
+  ansible.builtin.lineinfile:
+    create: true
+    line: "{{ kernelrelease.stdout }}"
+    mode: "u=rw,g=r,o=r"
+    path: "{{ bootlinux_artifacts_dir }}/kernel.release"
+
+- name: Skip the kernel install steps
+  ansible.builtin.meta: end_play
diff --git a/playbooks/roles/bootlinux/tasks/install-deps/redhat/main.yml b/playbooks/roles/bootlinux/tasks/install-deps/redhat/main.yml
index 7849f6f3..0f966d67 100644
--- a/playbooks/roles/bootlinux/tasks/install-deps/redhat/main.yml
+++ b/playbooks/roles/bootlinux/tasks/install-deps/redhat/main.yml
@@ -68,6 +68,24 @@
       - btrfs-progs
   when: ansible_distribution == 'Fedora'
 
+- name: Install rpmbuild
+  become: true
+  become_method: ansible.builtin.sudo
+  ansible.builtin.dnf:
+    name:
+      - elfutils-devel
+      - perl-core
+      - rpm-build
+      - rsync
+    state: present
+    update_cache: true
+  retries: 3
+  delay: 5
+  register: rpmbuild_result
+  until: rpmbuild_result is succeeded
+  when:
+    - bootlinux_builder|bool
+
 - name: Remove packages that mess with initramfs
   become: yes
   become_method: sudo
diff --git a/playbooks/roles/bootlinux/tasks/main.yml b/playbooks/roles/bootlinux/tasks/main.yml
index 62a081a9..57564399 100644
--- a/playbooks/roles/bootlinux/tasks/main.yml
+++ b/playbooks/roles/bootlinux/tasks/main.yml
@@ -80,6 +80,12 @@
   when:
     - bootlinux_targets|bool
 
+- name: Build kernel on the builder node
+  ansible.builtin.include_tasks:
+    file: "{{ role_path }}/tasks/build/builder.yml"
+  when:
+    - bootlinux_builder|bool
+
 - name: Run uname before
   command: "uname -r"
   register: uname_cmd_before
-- 
2.47.2