From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 064A827586F for ; Fri, 30 May 2025 17:52:37 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal:i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1748627557; cv=none; b=jLkrrmhQh9VYNaltthFilRr8Fsfv4N1PHft1PffZRRarfyS9coEdfJeLAX3e+NQJ+gdGLng9n/F6oVffhgbmYs0nxgLc8G+L9q+YFBU8gj0DQoP6XGE1lpuhRzE5BnQwCb26et/52DJpAJvel9oBMfAO5TKlsXrCmHlCJn2mkaM= ARC-Message-Signature:i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1748627557; c=relaxed/simple; bh=ARSKzGP4+3OV/rr67/Os7Rwdeqxqc7TE+/gVb7G9HcI=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=eSmmAr7KGkuSaC/cmWQgk25MJAfVIPTok3DgZSK1HX38ZLVnz6o7YqaPKtIYeyYloCMDJC9ldPxig0AViU0Xg6SLoaedcGsEcj7qM6ahyoG2NVVDi0/qODnbuSlei9AJfDX97+vWQbSkb88mCjyGNxqwwE5sbZMWN1IU5r9Mt5w= ARC-Authentication-Results:i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=FUdK4UP4; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="FUdK4UP4" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 9B438C4CEEE; Fri, 30 May 2025 17:52:36 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1748627556; bh=ARSKzGP4+3OV/rr67/Os7Rwdeqxqc7TE+/gVb7G9HcI=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=FUdK4UP4J8rQKrKnhCpDg3a6xouY1kjNxw2ekZLZHgFgar44Q292GaoLeLwD3tCM7 sUVVTzIVlzrP5pJTU6QfbRUvx51VlgFOSF2rHxcR9Myk7E+OOxZM7CsQtsj+Jq2Cfl annrZjkNvEj6uRHOuVBhQ8y5Cjg76fx2GKF5qsWXNxQYEzVYulNqR7b3Wp1m1hmDGy VbPemutSVc54QkpT2F9l/evd/dQ7hWoTzgbtjO6GwIb5ca8KQ/FmcwpoNlvqd8Ox5s iFcutGvHK4JnJitvvsN1AQkScMcg4vcQVrvhlhkVWomKh40n3/PHugTufIYNpfpRF6 Bs1JpUwp9TZCg== From: Chuck Lever To: Cc: Chuck Lever Subject: [PATCH v2 08/12] guestfs: Convert scripts/bringup_guestfs.sh to Ansible Date: Fri, 30 May 2025 13:52:25 -0400 Message-ID: <20250530175229.489925-9-cel@kernel.org> X-Mailer: git-send-email 2.49.0 In-Reply-To: <20250530175229.489925-1-cel@kernel.org> References: <20250530175229.489925-1-cel@kernel.org> Precedence: bulk X-Mailing-List: kdevops@lists.linux.dev List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: 8bit From: Chuck Lever The bringup_guestfs.sh is converted to Ansible to enable the virt-builder part of bring to run in parallel. This also gives us a fighting chance to integrate terraform and guestfs ssh key management in the future. In addition to those benefits, the new Ansible code takes a stab at being more idempotent than the shell script was. Idempotency is achieved by checking that each guest is defined, and then defining it if not; then using community.libvirt.virt to boot the guest, which does nothing if the guest is already running. Signed-off-by: Chuck Lever --- playbooks/guestfs.yml | 2 +- playbooks/roles/guestfs/defaults/main.yml | 1 + .../guestfs/tasks/bringup/extra-disks.yml | 16 +++ .../roles/guestfs/tasks/bringup/largeio.yml | 11 ++ .../roles/guestfs/tasks/bringup/main.yml | 132 ++++++++++++++++++ playbooks/roles/guestfs/tasks/main.yml | 29 ++++ scripts/guestfs.Makefile | 1 - 7 files changed, 190 insertions(+), 2 deletions(-) create mode 100644 playbooks/roles/guestfs/tasks/bringup/extra-disks.yml create mode 100644 playbooks/roles/guestfs/tasks/bringup/largeio.yml create mode 100644 playbooks/roles/guestfs/tasks/bringup/main.yml diff --git a/playbooks/guestfs.yml b/playbooks/guestfs.yml index 8bb496e0f05a..784775cdb818 100644 --- a/playbooks/guestfs.yml +++ b/playbooks/guestfs.yml @@ -1,6 +1,6 @@ --- - name: Provision target nodes with libvirt/guestfs - gather_facts: false + gather_facts: true connection: local hosts: all roles: diff --git a/playbooks/roles/guestfs/defaults/main.yml b/playbooks/roles/guestfs/defaults/main.yml index e250a40fd27a..eec137bd730a 100644 --- a/playbooks/roles/guestfs/defaults/main.yml +++ b/playbooks/roles/guestfs/defaults/main.yml @@ -2,3 +2,4 @@ distro_debian_based: false libvirt_uri_system: false +libvirt_enable_largeio: false diff --git a/playbooks/roles/guestfs/tasks/bringup/extra-disks.yml b/playbooks/roles/guestfs/tasks/bringup/extra-disks.yml new file mode 100644 index 000000000000..438b65ed5d3d --- /dev/null +++ b/playbooks/roles/guestfs/tasks/bringup/extra-disks.yml @@ -0,0 +1,16 @@ +--- +- name: Create the new drive image + ansible.builtin.command: + argv: + - "qemu-img" + - "create" + - "-f" + - "{{ libvirt_extra_drive_format }}" + - "{{ path }}" + - "100G" + +- name: Update the permission settings of the drive image file + ansible.builtin.file: + path: "{{ path }}" + group: "{{ libvirt_qemu_group }}" + mode: "g+rw,o-rw" diff --git a/playbooks/roles/guestfs/tasks/bringup/largeio.yml b/playbooks/roles/guestfs/tasks/bringup/largeio.yml new file mode 100644 index 000000000000..4246677d18d8 --- /dev/null +++ b/playbooks/roles/guestfs/tasks/bringup/largeio.yml @@ -0,0 +1,11 @@ +--- +- name: Compute the total number of devices to build + ansible.builtin.set_fact: + total_devices: "{{ libvirt_largeio_pow_limit * libvirt_largeio_drives_per_space }}" + +- name: Create largeio block devices + ansible.builtin.include_tasks: + file: "{{ role_path }}/tasks/extra_disks.yml" + vars: + path: "{{ storagedir }}/{{ inventory_hostname }}/extra{{ item }}.{{ libvirt_extra_drive_format }}" + loop: "{{ range(0, total_devices) | list }}" diff --git a/playbooks/roles/guestfs/tasks/bringup/main.yml b/playbooks/roles/guestfs/tasks/bringup/main.yml new file mode 100644 index 000000000000..007dabddb2b4 --- /dev/null +++ b/playbooks/roles/guestfs/tasks/bringup/main.yml @@ -0,0 +1,132 @@ +--- +- name: List defined libvirt guests + run_once: true + community.libvirt.virt: + command: list_vms + uri: "{{ libvirt_uri }}" + register: defined_vms + +- name: Provision each target node + when: + - 'inventory_hostname not in defined_vms.list_vms' + block: + - name: Set the pathname of the ssh directory for each target node + ansible.builtin.set_fact: + ssh_key_dir: "{{ guestfs_path }}/{{ inventory_hostname }}/ssh" + + - name: Set the pathname of the ssh key for each target node + ansible.builtin.set_fact: + ssh_key: "{{ ssh_key_dir }}/id_ed25519" + + - name: Generate ssh keys for each target node + block: + - name: Create the ssh key directory on the control host + ansible.builtin.file: + path: "{{ ssh_key_dir }}" + state: directory + mode: "u=rwx" + + - name: Generate fresh keys for each target node + ansible.builtin.command: + cmd: 'ssh-keygen -q -t ed25519 -f {{ ssh_key }} -N ""' + + - name: Set the pathname of the root disk image for each target node + ansible.builtin.set_fact: + root_image: "{{ storagedir }}/{{ inventory_hostname }}/root.raw" + + - name: Create the storage pool directory for each target node + ansible.builtin.file: + path: "{{ storagedir }}/{{ inventory_hostname }}" + state: directory + + - name: Duplicate the root disk image for each target node + ansible.builtin.command: + cmd: "cp --reflink=auto {{ base_image }} {{ root_image }}" + + - name: Get the timezone of the control host + ansible.builtin.command: + cmd: "timedatectl show -p Timezone --value" + register: host_timezone + + - name: Build the root image for each target node (as root) + become: true + become_method: ansible.builtin.sudo + ansible.builtin.command: + argv: + - "virt-sysprep" + - "-a" + - "{{ root_image }}" + - "--hostname" + - "{{ inventory_hostname }}" + - "--ssh-inject" + - "kdevops:file:{{ ssh_key }}.pub" + - "--timezone" + - "{{ host_timezone.stdout }}" + when: + - libvirt_uri_system|bool + + - name: Build the root image for each target node (non-root) + ansible.builtin.command: + argv: + - "virt-sysprep" + - "-a" + - "{{ root_image }}" + - "--hostname" + - "{{ inventory_hostname }}" + - "--ssh-inject" + - "kdevops:file:{{ ssh_key }}.pub" + - "--timezone" + - "{{ host_timezone.stdout }}" + when: + - not libvirt_uri_system|bool + + - name: Create largeio devices + ansible.builtin.include_tasks: + file: "{{ role_path }}/tasks/bringup/largeio.yml" + when: + - libvirt_enable_largeio|bool + + - name: Create extra disks + vars: + path: "{{ storagedir }}/{{ inventory_hostname }}/extra{{ item }}.{{ libvirt_extra_drive_format }}" + ansible.builtin.include_tasks: + file: "{{ role_path }}/tasks/bringup/extra-disks.yml" + loop: "{{ range(0, 4) | list }}" + when: + - not libvirt_enable_largeio|bool + + - name: Define each target node + vars: + xml_file: "{{ guestfs_path }}/{{ inventory_hostname }}/{{ inventory_hostname }}.xml" + community.libvirt.virt: + command: define + name: "{{ inventory_hostname }}" + xml: "{{ lookup('file', xml_file) }}" + uri: "{{ libvirt_uri }}" + + - name: Find PCIe passthrough devices + ansible.builtin.find: + paths: "{{ guestfs_path }}/{{ inventory_hostname }}" + file_type: file + patterns: "pcie_passthrough_*.xml" + register: passthrough_devices + + - name: Attach PCIe passthrough devices to each target node + ansible.builtin.command: + argv: + - "virsh" + - "attach-device" + - "{{ inventory_hostname }}" + - "{{ item }}" + - "--config" + loop: "{{ passthrough_devices.files }}" + loop_control: + label: "Doing PCI-E passthrough for device {{ item }}" + when: + - passthrough_devices.matched > 0 + +- name: Boot each target node + community.libvirt.virt: + name: "{{ inventory_hostname }}" + uri: "{{ libvirt_uri }}" + state: running diff --git a/playbooks/roles/guestfs/tasks/main.yml b/playbooks/roles/guestfs/tasks/main.yml index 6981d3c1a70b..8201c8a4dcb2 100644 --- a/playbooks/roles/guestfs/tasks/main.yml +++ b/playbooks/roles/guestfs/tasks/main.yml @@ -21,6 +21,35 @@ ansible.builtin.import_tasks: file: "{{role_path }}/tasks/bringup/network.yml" +- name: Set the pathname of storage pool directory + tags: + - bringup + ansible.builtin.set_fact: + storagedir: "{{ kdevops_storage_pool_path }}/guestfs" + +- name: Set the pathname of the OS base image + tags: + - bringup + ansible.builtin.set_fact: + base_image: "{{ storagedir }}/base_images/{{ virtbuilder_os_version }}.raw" + +- name: Ensure the base OS image exists + delegate_to: localhost + run_once: true + tags: + - bringup + ansible.builtin.import_role: + name: base_image + vars: + base_image_os_version: "{{ virtbuilder_os_version }}" + base_image_pathname: "{{ base_image }}" + +- name: Bring up each target node + tags: + - bringup + ansible.builtin.import_tasks: + file: "{{role_path }}/tasks/bringup/main.yml" + - name: Shut down and destroy each target node tags: - destroy diff --git a/scripts/guestfs.Makefile b/scripts/guestfs.Makefile index f2900a684960..0f0a8b858dba 100644 --- a/scripts/guestfs.Makefile +++ b/scripts/guestfs.Makefile @@ -79,7 +79,6 @@ bringup_guestfs: $(GUESTFS_BRINGUP_DEPS) -i hosts playbooks/guestfs.yml \ --extra-vars=@./extra_vars.yaml \ --tags bringup - $(Q)$(TOPDIR)/scripts/bringup_guestfs.sh $(Q)ansible-playbook $(ANSIBLE_VERBOSE) --connection=local \ --inventory localhost, \ playbooks/bringup_guestfs.yml \ -- 2.49.0