[RFC PATCH 3/3] Experimental: Add a separate install_linux role

[cel@kernel.org]

From: Chuck Lever <chuck.lever@oracle.com>

Add a role that can grab the artifacts in workflows/linux/artifacts
and install them on all guests/instances.

Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
---
 playbooks/install_linux.yml                   |   4 +
 playbooks/roles/install_linux/README.md       | 136 ++++++++++++
 .../roles/install_linux/defaults/main.yml     |  43 ++++
 .../tasks/install-deps/debian/main.yml        |  44 ++++
 .../tasks/install-deps/redhat/main.yml        |  76 +++++++
 .../tasks/install-deps/suse/main.yml          |  31 +++
 playbooks/roles/install_linux/tasks/main.yml  | 142 +++++++++++++
 .../tasks/update-grub/debian.yml              |   8 +
 .../tasks/update-grub/install.yml             | 196 ++++++++++++++++++
 .../install_linux/tasks/update-grub/main.yml  |  15 ++
 .../tasks/update-grub/redhat.yml              |  36 ++++
 .../install_linux/tasks/update-grub/suse.yml  |  11 +
 workflows/linux/Makefile                      |   8 +
 13 files changed, 750 insertions(+)
 create mode 100644 playbooks/install_linux.yml
 create mode 100644 playbooks/roles/install_linux/README.md
 create mode 100644 playbooks/roles/install_linux/defaults/main.yml
 create mode 100644 playbooks/roles/install_linux/tasks/install-deps/debian/main.yml
 create mode 100644 playbooks/roles/install_linux/tasks/install-deps/redhat/main.yml
 create mode 100644 playbooks/roles/install_linux/tasks/install-deps/suse/main.yml
 create mode 100644 playbooks/roles/install_linux/tasks/main.yml
 create mode 100644 playbooks/roles/install_linux/tasks/update-grub/debian.yml
 create mode 100644 playbooks/roles/install_linux/tasks/update-grub/install.yml
 create mode 100644 playbooks/roles/install_linux/tasks/update-grub/main.yml
 create mode 100644 playbooks/roles/install_linux/tasks/update-grub/redhat.yml
 create mode 100644 playbooks/roles/install_linux/tasks/update-grub/suse.yml

diff --git a/playbooks/install_linux.yml b/playbooks/install_linux.yml
new file mode 100644
index 000000000000..273ef4512f81
--- /dev/null
+++ b/playbooks/install_linux.yml
@@ -0,0 +1,4 @@
+---
+- hosts: all
+  roles:
+    - role: install_linux
diff --git a/playbooks/roles/install_linux/README.md b/playbooks/roles/install_linux/README.md
new file mode 100644
index 000000000000..0b440d2130b0
--- /dev/null
+++ b/playbooks/roles/install_linux/README.md
@@ -0,0 +1,136 @@
+bootlinux
+=========
+
+The ansible bootlinux lets you get, build and install Linux.  It also lets you
+apply custom patches, remove kernels, etc. Anything you have to do with regards
+to generic kernel development. The defaults it will track one of the latest
+stable kernels that are still supported, using the linux stable git tree.
+
+Requirements
+------------
+
+You are expected to have an extra partition
+
+Role Variables
+--------------
+
+  * infer_uid_and_group: defaults to False, if set to True, then we will ignore
+    the passed on data_user and data_group and instead try to infer this by
+    inspecting the `whoami` and getent on the logged in target system we are
+    provisioning. So if user sam is running able on a host, targetting a system
+    called foofighter and logging into that system using username pincho,
+    then the data_user will be set overwritten and set to pincho. We will then
+    also lookup for pincho's default group id and use that for data_group.
+    This is useful if you are targetting a slew of systems and don't really
+    want to deal with the complexities of the username and group, and the
+    default target username you use to ssh into a system suffices to use as
+    a base. This is set to False to remain compatible with old users of
+    this role.
+  * data_path: where to place the git trees we clone under
+  * data_user: the user to assign permissions to
+  * data_group: the group to assign permissions to
+
+  * data_device: the target device to use for the data partition
+  * data_fstype: the filesystem to store the data parition under
+  * data_label: the label to use
+  * data_fs_opts: the filesystem options to use, you want to ensure to add the
+    label
+
+  * target_linux_admin_name: your developer name
+  * target_linux_admin_email: your email
+  * target_linux_git: the git tree to clone, by default this is the linux-stable
+    tree
+  * target_linux_tree: the name of the tree
+  * target_linux_dir_path: where to place the tree on the target system
+
+  * target_linux_ref : the actual tag as used on linux, so v4.19.62
+  * target_linux_extra_patch: if defined an extra patch to apply with git
+     am prior to compilation
+  * target_linux_config: the configuration file to use
+  * make: the make command to use
+  * target_linux_make_cmd: the actual full make command and its arguments
+  * target_linux_make_install_cmd: the install command
+
+Dependencies
+------------
+
+None.
+
+Example Playbook
+----------------
+
+Below is an example playbook, say a bootlinux.yml file:
+
+```
+---
+- hosts: all
+  roles:
+    - role: bootlinux
+```
+
+Custom runs
+===========
+
+Say you want to boot compile a vanilla kernel and you have created a new
+section under the hosts file called [dev], with a subset of the [all] section.
+You can compile say a vanilla kernel v4.19.58 with an extra set of patches we'd
+`git am` for you on top by using the following:
+
+```
+cd ansible
+ansible-playbook -i hosts -l dev --extra-vars "target_linux_extra_patch=pend-v4.19.58-fixes-20190716-v2.patch" bootlinux.yml
+```
+
+You'd place the `pend-v4.19.58-fixes-20190716-v2.patch` file on the directory
+`ansible/roles/bootlinux/templates/`.
+
+Now say you wantd to be explicit about a tag of Linux you'd want to use:
+
+```
+ansible-playbook -i hosts -l dev --extra-vars "target_linux_ref=v4.19.21 "target_linux_extra_patch=try-v4.19.20-fixes-20190716-v1.patch" bootlinux.yml
+```
+
+To uninstall a kernel:
+
+```
+ansible-playbook -i hosts -l dev --tags uninstall-linux --extra-vars "uninstall_kernel_ver=4.19.58+" bootlinux.yml
+```
+
+To ensure you can get the grub prompt:
+
+```bash
+ansible-playbook -i hosts --tags console,vars,manual-update-grub playbooks/bootlinux.yml
+```
+
+The ansible bootlinux role relies on the create_partition role to create a data
+partition where we can stuff code, and compile it. To test that aspect of
+the bootlinux role you can run:
+
+```
+ansible-playbook -i hosts -l baseline --tags data_partition,partition bootlinux.yml
+```
+
+To reboot all hosts:
+
+```bash
+ansible-playbook -i hosts bootlinux.yml --tags reboot
+```
+
+For further examples refer to one of this role's users, the
+[https://github.com/mcgrof/kdevops](kdevops) project or the
+[https://github.com/mcgrof/oscheck](oscheck) project from where
+this code originally came from.
+
+# TODO
+
+## Avoiding carrying linux-next configs
+
+It seems a waste of space to be adding configurations for linux-next for all
+tags. It seems easier to just look for the latest linux-next and try that.
+We just symlink linux-next files when we really need to, and when something
+really needs a new config, we then just add a new file.
+
+License
+-------
+
+copyleft-next-0.3.1
diff --git a/playbooks/roles/install_linux/defaults/main.yml b/playbooks/roles/install_linux/defaults/main.yml
new file mode 100644
index 000000000000..43edb3cafd79
--- /dev/null
+++ b/playbooks/roles/install_linux/defaults/main.yml
@@ -0,0 +1,43 @@
+# SPDX-License-Identifier copyleft-next-0.3.1
+---
+kdevops_bootlinux: False
+infer_uid_and_group: False
+
+data_path: "/data"
+data_user: "vagrant"
+data_group: "vagrant"
+
+data_device: "/dev/disk/by-id/nvme-QEMU_NVMe_Ctrl_kdevops0"
+data_fstype: "xfs"
+data_label: "data"
+data_fs_opts: "-L {{ disk_setup_label }}"
+
+# Linux target defaults
+target_linux_admin_name: "Hacker Amanda"
+target_linux_admin_email: "devnull@kernel.org"
+target_linux_git: "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git"
+target_linux_shallow_depth: 0
+target_linux_tree: "linux-stable"
+target_linux_dir_path: "{{ data_path }}/{{ target_linux_tree }}"
+kdevops_baseline_and_dev: False
+
+target_linux_ref: "v4.19.133"
+target_linux_delta_file:
+target_linux_config: "config-{{ target_linux_ref }}"
+make: "make"
+# Once ansible v2.10 becomes available we can move on to using
+# ansible_processor_nproc but that was merged in 2020:
+# The commit is 34db57a47f875d11c4068567b9ec7ace174ec4cf
+# introduce fact "ansible_processor_nproc": number of usable vcpus #66569
+# https://github.com/ansible/ansible/pull/66569
+target_linux_make_cmd: "{{ make }} -j{{ ansible_processor_vcpus }}"
+target_linux_make_install_cmd: "{{ target_linux_make_cmd }} modules_install install"
+
+uninstall_kernel_enable: False
+
+build_artifacts_dir: "{{ topdir_path }}/workflows/linux/artifacts/"
+
+kdevops_workflow_enable_cxl: False
+
+bootlinux_cxl_test: False
+bootlinux_tree_set_by_cli: False
diff --git a/playbooks/roles/install_linux/tasks/install-deps/debian/main.yml b/playbooks/roles/install_linux/tasks/install-deps/debian/main.yml
new file mode 100644
index 000000000000..51b216e47b06
--- /dev/null
+++ b/playbooks/roles/install_linux/tasks/install-deps/debian/main.yml
@@ -0,0 +1,44 @@
+---
+# Install dependencies for building linux on Debian
+
+- name: Update apt cache
+  become: yes
+  become_method: sudo
+  apt:
+    update_cache: yes
+  tags: linux
+
+# apt-get build-dep does not capture all requirements
+- name: Install Linux kernel build dependencies
+  become: yes
+  become_method: sudo
+  apt:
+    name:
+      - bison
+      - flex
+      - git
+      - gcc
+      - make
+      - gawk
+      - bc
+      - dump
+      - indent
+      - sed
+      - libssl-dev
+      - libelf-dev
+      - liburcu-dev
+      - xfsprogs
+      - e2fsprogs
+      - btrfs-progs
+      - ntfs-3g
+      - mdadm
+      - rpcbind
+      - portmap
+      - hwinfo
+      - open-iscsi
+      - python3-pip
+      - zstd
+      - libncurses-dev
+      - b4
+    state: present
+  tags: linux
diff --git a/playbooks/roles/install_linux/tasks/install-deps/redhat/main.yml b/playbooks/roles/install_linux/tasks/install-deps/redhat/main.yml
new file mode 100644
index 000000000000..57a340979fcd
--- /dev/null
+++ b/playbooks/roles/install_linux/tasks/install-deps/redhat/main.yml
@@ -0,0 +1,76 @@
+---
+- name: Enable installation of packages from EPEL
+  ansible.builtin.include_role:
+    name: epel-release
+  when:
+    - ansible_distribution != "Fedora"
+
+- name: Install packages we care about
+  become: true
+  become_method: ansible.builtin.sudo
+  ansible.builtin.dnf:
+    update_cache: true
+    name: "{{ packages }}"
+  retries: 3
+  delay: 5
+  register: result
+  until: result is succeeded
+  vars:
+    packages:
+      - bison
+      - flex
+      - git-core
+      - e2fsprogs
+      - xfsprogs
+      - xfsdump
+      - lvm2
+      - gcc
+      - make
+      - gawk
+      - bc
+      - dump
+      - libtool
+      - psmisc
+      - sed
+      - vim
+      - fio
+      - libaio-devel
+      - diffutils
+      - net-tools
+      - ncurses-devel
+      - xfsprogs
+      - e2fsprogs
+      - elfutils-libelf-devel
+      - ntfs-3g
+      - mdadm
+      - rpcbind
+      - portmap
+      - hwinfo
+      - iscsi-initiator-utils
+      - openssl
+      - openssl-devel
+      - dwarves
+      - userspace-rcu
+      - zstd
+
+- name: Install btrfs-progs
+  become: true
+  become_method: ansible.builtin.sudo
+  ansible.builtin.dnf:
+    update_cache: true
+    name: "{{ packages }}"
+  retries: 3
+  delay: 5
+  register: result
+  until: result is succeeded
+  vars:
+    packages:
+      - btrfs-progs
+  when: ansible_distribution == 'Fedora'
+
+- name: Remove packages that mess with initramfs
+  become: true
+  become_method: ansible.builtin.sudo
+  ansible.builtin.dnf:
+    state: absent
+    name: dracut-config-generic
diff --git a/playbooks/roles/install_linux/tasks/install-deps/suse/main.yml b/playbooks/roles/install_linux/tasks/install-deps/suse/main.yml
new file mode 100644
index 000000000000..204a181b8237
--- /dev/null
+++ b/playbooks/roles/install_linux/tasks/install-deps/suse/main.yml
@@ -0,0 +1,31 @@
+---
+- name: Install Linux kernel build dependencies for SUSE sources
+  become: yes
+  become_method: sudo
+  zypper:
+    name:
+      - bison
+      - flex
+      - git-core
+      - gcc
+      - make
+      - gawk
+      - bc
+      - dump
+      - sed
+      - libopenssl-devel
+      - libelf-devel
+      - liburcu8
+      - diffutils
+      - net-tools
+      - ncurses-devel
+      - xfsprogs
+      - e2fsprogs
+      - btrfsprogs
+      - ntfs-3g
+      - mdadm
+      - rpcbind
+      - portmap
+      - hwinfo
+      - open-iscsi
+    disable_recommends: no
diff --git a/playbooks/roles/install_linux/tasks/main.yml b/playbooks/roles/install_linux/tasks/main.yml
new file mode 100644
index 000000000000..a0da4c110f6e
--- /dev/null
+++ b/playbooks/roles/install_linux/tasks/main.yml
@@ -0,0 +1,142 @@
+---
+- name: Include optional extra_vars
+  ansible.builtin.include_vars:
+    file: "{{ item }}"
+  with_first_found:
+    - files:
+        - "../extra_vars.yml"
+        - "../extra_vars.yaml"
+        - "../extra_vars.json"
+      skip: true
+  failed_when: false
+  tags: vars
+
+- name: Debian-specific set up
+  ansible.builtin.import_tasks: install-deps/debian/main.yml
+  when:
+    - ansible_os_family == "Debian"
+
+- name: Suse-specific set up
+  ansible.builtin.import_tasks: install-deps/suse/main.yml
+  when:
+    - ansible_os_family == "Suse"
+
+- name: Red Hat-specific set up
+  ansible.builtin.import_tasks: install-deps/redhat/main.yml
+  when:
+    - ansible_os_family == "RedHat"
+
+# We use "console serial" so to enable real consoles to be
+# preferred first, and fallback to the serial as secondary
+# option. This let's us work with hardware serial consoles
+# say on IPMIs and virtual guests ('virsh console').
+- name: Ensure we can get the GRUB prompt on reboot
+  become: true
+  become_flags: 'su - -c'
+  become_method: ansible.builtin.sudo
+  ansible.builtin.lineinfile:
+    path: /etc/default/grub
+    regexp: '^GRUB_TERMINAL='
+    line: GRUB_TERMINAL="console serial"
+  tags:
+    - linux
+    - git
+    - config
+    - console
+
+- name: Update the boot GRUB file
+  ansible.builtin.import_tasks: update-grub/main.yml
+  tags:
+    - linux
+    - uninstall-linux
+    - manual-update-grub
+    - console
+
+- name: Ensure DEFAULTDEBUG is set
+  become: true
+  become_flags: 'su - -c'
+  become_method: ansible.builtin.sudo
+  register: grub_default_saved_cmd
+  ansible.builtin.lineinfile:
+    path: /etc/sysconfig/kernel
+    regexp: '^DEFAULTDEBUG='
+    line: DEFAULTDEBUG=yes
+  when:
+    - ansible_os_family == "RedHat"
+  tags:
+    - linux
+    - git
+    - config
+    - saved
+
+- name: Install the built kernel RPMs on the target nodes
+  when:
+    - ansible_os_family == "RedHat"
+  tags:
+    - linux
+    - install-linux
+  block:
+    - name: Find the kernel build artifacts on the control host
+      delegate_to: localhost
+      ansible.builtin.find:
+        paths: "{{ build_artifacts_dir }}"
+        patterns: "*.rpm"
+        file_type: file
+        recurse: true
+      register: found_rpms
+
+    - name: Upload the kernel build artifacts to the target nodes
+      ansible.builtin.copy:
+        src: "{{ item.path }}"
+        dest: "/tmp"
+        mode: "u=rw,g=r,o=r"
+      loop: "{{ found_rpms.files }}"
+      loop_control:
+        label: "Uploading {{ item.path }}"
+
+    - name: Initialize list of packages to install
+      ansible.builtin.set_fact:
+        packages: []
+
+    - name: Build a list of packages to install
+      ansible.builtin.set_fact:
+        packages: "{{ packages + ['/tmp/' + item.path | basename ] }}"
+      loop: "{{ found_rpms.files }}"
+      loop_control:
+        label: "Adding {{ item.path }}"
+
+    - name: Install the kernel build artifacts on the target nodes
+      become: true
+      become_method: ansible.builtin.sudo
+      ansible.builtin.dnf:
+        name: "{{ packages }}"
+        state: present
+        disable_gpg_check: true
+
+- name: Set the default kernel on the target nodes
+  ansible.builtin.import_tasks: update-grub/install.yml
+  tags:
+    - linux
+    - git
+    - config
+    - saved
+
+- name: Reboot the target nodes into Linux {{ target_linux_tree }}
+  become: true
+  become_method: ansible.builtin.sudo
+  ansible.builtin.reboot:
+  tags:
+    - linux
+    - reboot
+
+- name: Refresh facts
+  ansible.builtin.gather_facts:
+
+- name: Check the uname on the target nodes
+  ansible.builtin.debug:
+    msg: "Target kernel {{ target_linux_ref }}; Running kernel {{ ansible_kernel }}"
+  tags:
+    - linux
+    - git
+    - config
+    - uname
diff --git a/playbooks/roles/install_linux/tasks/update-grub/debian.yml b/playbooks/roles/install_linux/tasks/update-grub/debian.yml
new file mode 100644
index 000000000000..3c7deea2161a
--- /dev/null
+++ b/playbooks/roles/install_linux/tasks/update-grub/debian.yml
@@ -0,0 +1,8 @@
+- name: Run update-grub
+  become: yes
+  become_flags: 'su - -c'
+  become_method: sudo
+  command: "update-grub"
+  register: grub_update
+  changed_when: "grub_update.rc == 0"
+  tags: [ 'linux', 'manual-update-grub', 'console' ]
diff --git a/playbooks/roles/install_linux/tasks/update-grub/install.yml b/playbooks/roles/install_linux/tasks/update-grub/install.yml
new file mode 100644
index 000000000000..17966af58210
--- /dev/null
+++ b/playbooks/roles/install_linux/tasks/update-grub/install.yml
@@ -0,0 +1,196 @@
+# There is slightly confusing user-experience and not complete documentation
+# about the requirements for using grub-set-default in light of the fact that
+# most Linux distributions use sub-menus. You need to use GRUB_DEFAULT=saved
+# there is a few caveats with its use which are not well documented anywhere
+# and I'm pretty sure tons of people are running into these issues.
+#
+# I'll document them here for posterity and so to justify the approach used
+# in kdevops to ensure we do boot the correct kernel.
+#
+# Some users erroneously claim that you also need GRUB_SAVEDEFAULT=true when
+# using GRUB_DEFAULT=saved but this is not true. The issue with using
+# GRUB_DEFAULT=saved which causes confusion is that most distributions
+# today use submenus folks do not take these into account when using
+# grub-set-default and the documentation about grub-set-default is not
+# clear about this requirement.
+#
+# Sadly, if you use a bogus kernel grub-set-default will not complain. For
+# example since most distributions use submenus, if you install a new kernel you
+# may end up in a situation as follows:
+#
+# menuentry 'Debian GNU/Linux' ... {
+#   ...
+# }
+# submenu 'Advanced options for Debian GNU/Linux' ... {
+#   menuentry 'Debian GNU/Linux, with Linux 5.16.0-4-amd64' ... {
+#     ...
+#   }
+#   menuentry 'Debian GNU/Linux, with Linux 5.16.0-4-amd64 (recovery mode)' ... {
+#     ...
+#   }
+#   menuentry 'Debian GNU/Linux, with Linux 5.10.105' ... {
+#     ...
+#   }
+#   ... etc ...
+# }
+#
+# So under this scheme the 5.10.105 kernel is actually "1>2" and so if
+# you used:
+#
+#   grub-set-default 3
+#
+# This would not return an error and you would expect it to work. This
+# is a bug in grub-set-default, it should return an error. The correct
+# way to set this with submenus would be:
+#
+#   grub-set-default "1>2"
+#
+# However doing the reverse mapping is something which can get complicated
+# and there is no upstream GRUB2 support to do this for you. We can simplify
+# this problem instead by disabling the submenus, with GRUB_DISABLE_SUBMENU=y,
+# making the menu flat and then just querying for the linear mapping using
+# ansible using awk | grep and tail.
+#
+# So for instance, using GRUB_DISABLE_SUBMENU=y results in the following
+# options:
+#
+# vagrant@kdevops-xfs-nocrc ~ $ awk -F\' '/menuentry / {print $2}' /boot/grub/grub.cfg |  awk '{print NR-1" ... "$0}'
+# 0 ... Debian GNU/Linux, with Linux 5.16.0-4-amd64
+# 1 ... Debian GNU/Linux, with Linux 5.16.0-4-amd64 (recovery mode)
+# 2 ... Debian GNU/Linux, with Linux 5.10.105
+# 3 ... Debian GNU/Linux, with Linux 5.10.105 (recovery mode)
+# 4 ... Debian GNU/Linux, with Linux 5.10.0-5-amd64
+# 5 ... Debian GNU/Linux, with Linux 5.10.0-5-amd64 (recovery mode)
+#
+# We have a higher degree of confidence with this structure when looking
+# for "5.10.105" that its respective boot entry 2 is the correct one. So we'd
+# now just use:
+#
+#   grub-set-default 2
+- name: Ensure we have GRUB_DEFAULT=saved
+  become: true
+  become_flags: 'su - -c'
+  become_method: ansible.builtin.sudo
+  ansible.builtin.lineinfile:
+    path: /etc/default/grub
+    regexp: '^GRUB_DEFAULT='
+    line: GRUB_DEFAULT=saved
+  register: grub_default_saved_cmd
+  tags:
+    - linux
+    - git
+    - config
+    - saved
+
+- name: Use GRUB_DISABLE_SUBMENU=y to enable grub-set-default use with one digit
+  become: true
+  become_flags: 'su - -c'
+  become_method: ansible.builtin.sudo
+  register: grub_disable_submenu_cmd
+  ansible.builtin.lineinfile:
+    path: /etc/default/grub
+    regexp: '^GRUB_DISABLE_SUBMENU='
+    line: GRUB_DISABLE_SUBMENU=y
+  tags:
+    - linux
+    - git
+    - config
+    - saved
+
+- name: Update your boot GRUB file if necessary to ensure GRUB flat earth
+  ansible.builtin.import_tasks: update-grub/main.yml
+  tags:
+    - linux
+    - uninstall-linux
+    - manual-update-grub
+    - console
+
+- name: Read the artifacts release file
+  delegate_to: localhost
+  vars:
+    release_file: "{{ topdir_path }}/workflows/linux/artifacts/kernel.release"
+  ansible.builtin.set_fact:
+    kernelrelease: "{{ lookup('file', release_file) }}"
+
+- name: Show kernel release information
+  ansible.builtin.debug:
+    var: kernelrelease
+
+- name: Construct command line to determine default kernel ID
+  ansible.builtin.set_fact:
+    determine_default_kernel_id: >-
+      awk -F\' '/menuentry / {print $2}'
+      /boot/grub/grub.cfg | awk '{print NR-1" ... "$0}' |
+      grep {{ kernelrelease }} | head -1 | awk '{print $1}'
+  when:
+    - ansible_os_family != 'RedHat' or ansible_distribution_major_version | int < 8
+
+- name: Construct command line to determine default kernel ID for RHEL >= 8
+  ansible.builtin.set_fact:
+    determine_default_kernel_id: >-
+      for f in $(ls -1 /boot/loader/entries/*.conf); do
+      cat $f;
+      done | grep title | awk '{ gsub("title ", "", $0); print }' | grep '{{ kernelrelease }}';
+  when:
+    - ansible_os_family == "RedHat" and ansible_distribution_major_version | int >= 8
+
+# If this fails then grub-set-default won't be run, and the assumption here
+# is either you do the work to enhance the heuristic or live happy with the
+# assumption that grub2's default of picking the latest kernel is the best
+# option.
+- name: Try to find your target kernel's GRUB boot entry number now that the menu is flattened for {{ target_linux_ref }} using inferred KERNELRELEASE {{ kernelrelease}}
+  become: true
+  become_flags: 'su - -c'
+  become_method: ansible.builtin.sudo
+  vars:
+    target_kernel: "{{ target_linux_ref | replace('v', '') }}"
+  ansible.builtin.shell: " {{ determine_default_kernel_id }} "
+  register: grub_boot_number_cmd
+  tags:
+    - linux
+    - git
+    - config
+    - saved
+
+- name: Obtain command to set default kernel to boot
+  ansible.builtin.set_fact:
+    grub_set_default_boot_kernel: grub-set-default
+  when:
+    - ansible_os_family != "RedHat" or ansible_distribution_major_version | int < 8
+
+- name: Obtain command to set default kernel to boot for RHEL >= 8
+  ansible.builtin.set_fact:
+    grub_set_default_boot_kernel: grub2-set-default
+  when:
+    - ansible_os_family == "RedHat" and ansible_distribution_major_version | int >= 8
+
+- name: Set the target kernel to be booted by default
+  become: true
+  become_flags: 'su - -c'
+  become_method: ansible.builtin.sudo
+  vars:
+    target_boot_entry: "{{ grub_boot_number_cmd.stdout_lines.0 }}"
+  ansible.builtin.command: "{{ grub_set_default_boot_kernel }} \"{{ target_boot_entry }}\""
+  when:
+    - grub_boot_number_cmd.rc == 0
+    - grub_boot_number_cmd.stdout != ""
+  tags:
+    - linux
+    - git
+    - config
+    - saved
+
+- name: Itemize kernel and GRUB entry we just selected
+  vars:
+    target_kernel: "{{ target_linux_ref | replace('v', '') }}"
+    target_boot_entry: "{{ grub_boot_number_cmd.stdout_lines.0 }}"
+  ansible.builtin.debug:
+    msg: "{{ target_kernel }} determined to be {{ target_boot_entry }} on the GRUB2 flat menu. Ran: grub-set-default {{ target_boot_entry }}"
+  when:
+    - grub_boot_number_cmd.rc == 0
+    - grub_boot_number_cmd.stdout != ""
+  tags:
+    - linux
+    - git
+    - config
+    - saved
diff --git a/playbooks/roles/install_linux/tasks/update-grub/main.yml b/playbooks/roles/install_linux/tasks/update-grub/main.yml
new file mode 100644
index 000000000000..a565e0ac26ac
--- /dev/null
+++ b/playbooks/roles/install_linux/tasks/update-grub/main.yml
@@ -0,0 +1,15 @@
+---
+- name: Debian-specific grub update
+  ansible.builtin.import_tasks: debian.yml
+  when:
+    - ansible_os_family == "Debian"
+
+- name: Red Hat-specific grub update
+  ansible.builtin.import_tasks: redhat.yml
+  when:
+    - ansible_os_family == "RedHat"
+
+- name: Suse-specific grub update
+  ansible.builtin.import_tasks: suse.yml
+  when:
+    - ansible_os_family == "Suse"
diff --git a/playbooks/roles/install_linux/tasks/update-grub/redhat.yml b/playbooks/roles/install_linux/tasks/update-grub/redhat.yml
new file mode 100644
index 000000000000..11a92f34bab6
--- /dev/null
+++ b/playbooks/roles/install_linux/tasks/update-grub/redhat.yml
@@ -0,0 +1,36 @@
+- name: Disable Grub menu auto-hide
+  become: true
+  become_flags: 'su - -c'
+  become_method: ansible.builtin.sudo
+  ansible.builtin.command: "grub2-editenv - unset menu_auto_hide"
+  register: grub_edit
+  changed_when: "grub_edit.rc == 0"
+
+- name: Determine if system was booted using UEFI
+  ansible.builtin.stat:
+    path: "/sys/firmware/efi/efivars"
+  register: efi_boot
+
+- name: Use /etc/grub2.cfg as the grub configuration file
+  ansible.builtin.set_fact:
+    grub_config_file: "/etc/grub2.cfg"
+  when:
+    - not efi_boot.stat.exists
+
+- name: Use /etc/grub2-efi.cfg as the configuration file
+  ansible.builtin.set_fact:
+    grub_config_file: "/etc/grub2-efi.cfg"
+  when:
+    - efi_boot.stat.exists
+
+- name: Run update-grub
+  become: true
+  become_flags: 'su - -c'
+  become_method: ansible.builtin.sudo
+  ansible.builtin.command: "grub2-mkconfig -o {{ grub_config_file }}"
+  register: grub_update
+  changed_when: "grub_update.rc == 0"
+  tags:
+    - linux
+    - manual-update-grub
+    - console
diff --git a/playbooks/roles/install_linux/tasks/update-grub/suse.yml b/playbooks/roles/install_linux/tasks/update-grub/suse.yml
new file mode 100644
index 000000000000..1cb1fdc55f58
--- /dev/null
+++ b/playbooks/roles/install_linux/tasks/update-grub/suse.yml
@@ -0,0 +1,11 @@
+- name: Run update-grub
+  become: true
+  become_flags: 'su - -c'
+  become_method: ansible.builtin.sudo
+  ansible.builtin.command: "update-bootloader --refresh"
+  register: grub_update
+  changed_when: "grub_update.rc == 0"
+  tags:
+    - linux
+    - manual-update-grub
+    - console
diff --git a/workflows/linux/Makefile b/workflows/linux/Makefile
index bb7441e71fda..06722d5903b3 100644
--- a/workflows/linux/Makefile
+++ b/workflows/linux/Makefile
@@ -85,6 +85,7 @@ linux-help-menu:
 	@echo "linux-grub-setup   - Ensures the appropriate target kernel is set to boot"
 	@echo "linux-reboot       - Reboot guests"
 	@echo "linux-packages     - Clones, builds, and packages a Linux kernel"
+	@echo "linux-artifacts    - Installs artifacts generated by 'linux-packages'"
 	@echo "uname              - Prints current running kernel"
 
 PHONY += linux-help-end
@@ -166,6 +167,13 @@ linux-packages:
 		$(KDEVOPS_PLAYBOOKS_DIR)/build_linux.yml \
 		--extra-vars="$(BOOTLINUX_ARGS)" $(LIMIT_HOSTS)
 
+PHONY += linux-artifacts
+linux-artifacts:
+	$(Q)ansible-playbook $(ANSIBLE_VERBOSE) \
+		-i $(KDEVOPS_HOSTFILE) \
+		$(KDEVOPS_PLAYBOOKS_DIR)/install_linux.yml \
+		--extra-vars="$(BOOTLINUX_ARGS)" $(LIMIT_HOSTS)
+
 PHONY += uname
 uname:
 	$(Q)ansible all -i hosts -b -m command -a "uname -r" -o \
-- 
2.49.0