From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from bombadil.infradead.org (bombadil.infradead.org [198.137.202.133]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id E36FC19C560 for ; Mon, 28 Jul 2025 01:14:37 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=198.137.202.133 ARC-Seal:i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1753665282; cv=none; b=R7veyqkN2RK4aFwTnXFQh6KyT+d2iHoQGBB19gCLg1bl8uRKCpYBOIR/fjbsbbp8jpXTqtYtbJY57fNQI39gcJrQhdsHo3wHLCyUs6awrJr1dBH47wrbJgBRtdrOyfOyVNnB+KzQ4IXFQsOOycUkvgJC7ugEYwSra3H813JYFy8= ARC-Message-Signature:i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1753665282; c=relaxed/simple; bh=WOplcb2givrspuOzPiGlwyJejhTIHCDTWpvW10PYd0Q=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=MGb1TPJ5iPLYJi2OG28b1HypEVb+dGkIfHgXCTW8lgRUf2dkZv4kuUbN/sVtUiBHUMP2NHaESDvqgaBPacYGmKPxyT4p2+8q1HRSYnqvI3rPZ/W2HGUae75AHpIgkjik1B9LQSK2dLhZblo+N1JtL0cmchh90lPCzGSVHjIWuZw= ARC-Authentication-Results:i=1; smtp.subspace.kernel.org; dmarc=fail (p=quarantine dis=none) header.from=kernel.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=qAOgcYGc; arc=none smtp.client-ip=198.137.202.133 Authentication-Results: smtp.subspace.kernel.org; dmarc=fail (p=quarantine dis=none) header.from=kernel.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="qAOgcYGc" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=bombadil.20210309; h=Sender:Content-Transfer-Encoding: MIME-Version:References:In-Reply-To:Message-ID:Date:Subject:Cc:To:From: Reply-To:Content-Type:Content-ID:Content-Description; bh=I6CEiNyqYB6q4mIlsGD4g3I3Fz5owc0vTK1ERpt6I/s=; b=qAOgcYGcAQLETySynmodlEbglP Zo43Wf3Uyrt085tU+YStypfTjza39kfZTRUMP7VO2WViQutMAi/UO8CB4dZ5Yxr9wUL79zl4QpDuc Zsk0Tio3fEY56z0bmEFK5LS+vig2ZQpjxWwLNAdbw0hUFzLgmrnhseHXx1G1svwtfT6zmN9Z4Sd2R Dw7+KmlrM5c9oVSZxnKvtrGIm/DnFGatI8XB204KSN+uURxGeKZZk9VRFga+j7R64mmFqbwZKOD6T VNvx0ozmO8rBywnMlxSgbtPO9rWNn0d0LTHQew9MjVjpZFv+x/iO6fuqB6+WCaAxENWQZsdeiJS+5 LFzXCO6A==; Received: from mcgrof by bombadil.infradead.org with local (Exim 4.98.2 #2 (Red Hat Linux)) id 1ugCRo-0000000DPko-1dLK; Mon, 28 Jul 2025 01:14:36 +0000 From: Luis Chamberlain To: Chuck Lever , Daniel Gomez , kdevops@lists.linux.dev Cc: Chuck Lever Subject: [PATCH v2 24/33] bootlinux: Add a new builder choice Date: Sun, 27 Jul 2025 18:14:24 -0700 Message-ID: <20250728011434.3197091-25-mcgrof@kernel.org> X-Mailer: git-send-email 2.49.0 In-Reply-To: <20250728011434.3197091-1-mcgrof@kernel.org> References: <20250728011434.3197091-1-mcgrof@kernel.org> Precedence: bulk X-Mailing-List: kdevops@lists.linux.dev List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Sender: Luis Chamberlain From: Chuck Lever Currently, for cloud configurations or when 9p is disabled, kdevops builds the test kernel on each test runner. This is inefficient, and gets worse as the test matrix for a single kernel version scales out. Instead we want to build the test kernel once and make those build artifacts available for test runners to install. This would be similar to what KOTD does now, except it does not require setting up a separate yum repo. Signed-off-by: Chuck Lever --- playbooks/roles/bootlinux/defaults/main.yml | 2 + .../roles/bootlinux/tasks/build/builder.yml | 228 ++++++++++++++++++ .../tasks/install-deps/redhat/main.yml | 18 ++ playbooks/roles/bootlinux/tasks/main.yml | 6 + 4 files changed, 254 insertions(+) create mode 100644 playbooks/roles/bootlinux/tasks/build/builder.yml diff --git a/playbooks/roles/bootlinux/defaults/main.yml b/playbooks/roles/bootlinux/defaults/main.yml index 4a1ad72e..50ce96bf 100644 --- a/playbooks/roles/bootlinux/defaults/main.yml +++ b/playbooks/roles/bootlinux/defaults/main.yml @@ -46,3 +46,5 @@ kdevops_workflow_enable_cxl: False bootlinux_cxl_test: False bootlinux_tree_set_by_cli: False + +bootlinux_artifacts_dir: "{{ topdir_path }}/workflows/linux/artifacts" diff --git a/playbooks/roles/bootlinux/tasks/build/builder.yml b/playbooks/roles/bootlinux/tasks/build/builder.yml new file mode 100644 index 00000000..c8363b8e --- /dev/null +++ b/playbooks/roles/bootlinux/tasks/build/builder.yml @@ -0,0 +1,228 @@ +--- +- name: Install b4 + become: true + become_method: ansible.builtin.sudo + ansible.builtin.pip: + name: + - b4 + when: + - target_linux_install_b4 is defined + - target_linux_install_b4 + - ansible_os_family == "Debian" + +- name: Clone {{ target_linux_tree }} + ansible.builtin.git: + repo: "{{ target_linux_git }}" + dest: "{{ target_linux_dir_path }}" + update: true + depth: "{{ target_linux_shallow_depth }}" + version: "{{ target_linux_ref }}" + register: result + retries: 3 + delay: 5 + until: result is succeeded + +- name: Copy the kernel delta to the builder + ansible.builtin.template: + src: "{{ target_linux_extra_patch }}" + dest: "{{ target_linux_dir_path }}/{{ target_linux_extra_patch }}" + owner: "{{ data_user }}" + group: "{{ data_group }}" + mode: "u=rw,g=r,o=r" + when: + - target_linux_extra_patch is defined + +- name: Apply the kernel delta on the builder + # noqa: command-instead-of-module + ansible.builtin.command: + cmd: "git am {{ target_linux_extra_patch }}" + chdir: "{{ target_linux_dir_path }}" + register: git_am + changed_when: not git_am.failed + when: + - target_linux_extra_patch is defined + +- name: Check git user name and email configuration + when: + - target_linux_apply_patch_message_id is defined + - target_linux_apply_patch_message_id | length > 0 + - bootlinux_b4_am_this_host|bool + block: + - name: Get the user's git config info + community.general.git_config_info: + scope: global + register: git_user_info + + - name: Set dummy git user email address + community.general.git_config: + name: user.email + scope: global + value: "user@example.com" + when: + - '"user.email" not in git_user_info.config_values' + + - name: Set dummy git user name + community.general.git_config: + name: user.name + scope: global + value: "Kdevops User" + when: + - '"user.name" not in git_user_info.config_values' + +- name: Show the message_id + ansible.builtin.debug: + msg: "{{ target_linux_apply_patch_message_id }}" + when: + - target_linux_apply_patch_message_id is defined + +- name: Apply a message patch set + ansible.builtin.shell: + chdir: "{{ target_linux_dir_path }}" + cmd: | + set -o pipefail + b4 am -o - {{ target_linux_apply_patch_message_id }} | git am + register: b4_am + changed_when: not b4_am.failed + when: + - target_linux_apply_patch_message_id is defined + - target_linux_apply_patch_message_id | length > 0 + - bootlinux_b4_am_this_host|bool + +- name: Copy the configuration for kernel {{ target_linux_tree }} + ansible.builtin.template: + src: "{{ role_path }}/templates/{{ linux_config }}" + dest: "{{ target_linux_dir_path }}/.config" + owner: "{{ data_user }}" + group: "{{ data_group }}" + mode: "u=rw,g=r,o=r" + +- name: Set the kernel localversion + ansible.builtin.lineinfile: + path: "{{ target_linux_dir_path }}/localversion" + line: "{{ target_linux_localversion }}" + mode: "u=rw,g=r,o=r" + create: true + when: + - target_linux_localversion is defined + - target_linux_localversion != "" + +- name: Configure kernel {{ target_linux_tree }} + community.general.make: + chdir: "{{ target_linux_dir_path }}" + target: "olddefconfig" + +- name: Build {{ target_linux_tree }} + community.general.make: + chdir: "{{ target_linux_dir_path }}" + jobs: "{{ ansible_processor_nproc }}" + target: "all" + +- name: Remove the old artifacts directory on the control host + delegate_to: localhost + ansible.builtin.file: + path: "{{ bootlinux_artifacts_dir }}" + state: absent + +- name: Ensure an empty artifacts directory exists on the control host + delegate_to: localhost + run_once: true + ansible.builtin.file: + path: "{{ bootlinux_artifacts_dir }}" + state: directory + mode: "u=rwx,g=rx,o=rx" + +- name: Build kernel .deb packages + when: + - ansible_os_family == "Debian" + block: + - name: Make the bindeb-pkg target + community.general.make: + chdir: "{{ target_linux_dir_path }}" + jobs: "{{ ansible_processor_nproc }}" + target: "bindeb-pkg" + + - name: Find the build artifacts + ansible.builtin.find: + paths: "{{ artifact_paths }}" + patterns: "*.deb" + file_type: file + recurse: true + register: found_debs + + - name: Fetch the build artifacts to the control host + ansible.builtin.fetch: + src: "{{ item.path }}" + dest: "{{ bootlinux_artifacts_dir }}/" + flat: true + loop: "{{ found_debs.files }}" + loop_control: + label: "Fetching {{ item.path | basename }} ..." + +- name: Build kernel .rpm packages + when: + - ansible_os_family != "Debian" + block: + - name: Build the list of artifacts directories on the builder + ansible.builtin.set_fact: + artifact_paths: + - "{{ target_linux_dir_path }}/rpmbuild/RPMS" + - "{{ ansible_env.HOME }}/rpmbuild/RPMS" + + - name: Wipe the artifact directories on the builder + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: "{{ artifact_paths }}" + loop_control: + label: "Deleting {{ item }} ..." + + - name: Create empty artifact directories on the builder + ansible.builtin.file: + path: "{{ item }}" + state: directory + mode: "u=rwx,g=rx,o=rx" + loop: "{{ artifact_paths }}" + loop_control: + label: "Creating {{ item }} ..." + + - name: Make the binrpm-pkg target + community.general.make: + chdir: "{{ target_linux_dir_path }}" + jobs: "{{ ansible_processor_nproc }}" + target: "binrpm-pkg" + params: + RPMOPTS: '--without devel' + + - name: Find the build artifacts + ansible.builtin.find: + paths: "{{ artifact_paths }}" + patterns: "*.rpm" + file_type: file + recurse: true + register: found_rpms + + - name: Fetch the build artifacts to the control host + ansible.builtin.fetch: + src: "{{ item.path }}" + dest: "{{ bootlinux_artifacts_dir }}/" + flat: true + loop: "{{ found_rpms.files }}" + loop_control: + label: "Fetching {{ item.path | basename }} ..." + +- name: Extract the release information of the built kernel + community.general.make: + chdir: "{{ target_linux_dir_path }}" + target: "kernelrelease" + register: kernelrelease + +- name: Store the kernel release information with the build artifacts + delegate_to: localhost + ansible.builtin.lineinfile: + create: true + line: "{{ kernelrelease.stdout }}" + mode: "u=rw,g=r,o=r" + path: "{{ bootlinux_artifacts_dir }}/kernel.release" + +- name: Skip the kernel install steps + ansible.builtin.meta: end_play diff --git a/playbooks/roles/bootlinux/tasks/install-deps/redhat/main.yml b/playbooks/roles/bootlinux/tasks/install-deps/redhat/main.yml index 7849f6f3..0f966d67 100644 --- a/playbooks/roles/bootlinux/tasks/install-deps/redhat/main.yml +++ b/playbooks/roles/bootlinux/tasks/install-deps/redhat/main.yml @@ -68,6 +68,24 @@ - btrfs-progs when: ansible_distribution == 'Fedora' +- name: Install rpmbuild + become: true + become_method: ansible.builtin.sudo + ansible.builtin.dnf: + name: + - elfutils-devel + - perl-core + - rpm-build + - rsync + state: present + update_cache: true + retries: 3 + delay: 5 + register: rpmbuild_result + until: rpmbuild_result is succeeded + when: + - bootlinux_builder|bool + - name: Remove packages that mess with initramfs become: yes become_method: sudo diff --git a/playbooks/roles/bootlinux/tasks/main.yml b/playbooks/roles/bootlinux/tasks/main.yml index 62a081a9..57564399 100644 --- a/playbooks/roles/bootlinux/tasks/main.yml +++ b/playbooks/roles/bootlinux/tasks/main.yml @@ -80,6 +80,12 @@ when: - bootlinux_targets|bool +- name: Build kernel on the builder node + ansible.builtin.include_tasks: + file: "{{ role_path }}/tasks/build/builder.yml" + when: + - bootlinux_builder|bool + - name: Run uname before command: "uname -r" register: uname_cmd_before -- 2.47.2