From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 6C73928AAF9 for ; Thu, 22 May 2025 13:31:42 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal:i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1747920702; cv=none; b=JeSONxGt5ulcXrdk3JpGlkbCicy+SNIQ2oMTgYb/+hCB2a3zbJ6IPlGP7zJ4sN6898bwqPAXV2XPmNV0NXMcdtp2xsUJBr9GrgWTb+JlQ3O8iYOKOJYOLJu/pYQ5fbi4Y62Psfiq2gKaDRW15ydXF0bCWElFfNUkT4h4IqwTF9o= ARC-Message-Signature:i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1747920702; c=relaxed/simple; bh=DxALtSOqM+nJill6BCI6VbMEhtXrWVgBCEkdOIsxxn0=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=njKtosTXkI11MoiOlUDMFBz49HEI8BEYmwPtK0EUiOLafGxjhTtYpgdb7fUrbZyCxyEweeEf4SLhkVMigiDMZjhg48g3e9HVd4XeWvLGUT93HhTV6uTZP1Kh4x6hnctjP2LkTckMctw8NmcLJSbOBc55AMlmFGbMcOMVfIK+UXA= ARC-Authentication-Results:i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=Pbt7nusc; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="Pbt7nusc" Received: by smtp.kernel.org (Postfix) with ESMTPSA id BC0DDC4CEF1; Thu, 22 May 2025 13:31:41 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1747920702; bh=DxALtSOqM+nJill6BCI6VbMEhtXrWVgBCEkdOIsxxn0=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=Pbt7nuscRhTJwwu720SSzku2g63yI9Si7iuWhk8n+hplx3cbd7fTgswO3378NejYx 4GEP/rSNBGsztGBx6L8eMgD6fxXzYd0roQC10jEP53GstH+CDMgsBChhdjG5x0vSy3 ai9l1HVFNpHtjei0NzRhWo6dqCTHWc3h/cxkMt6kDH2VVIlf17M2EiTleRoh1XmJaB Z5UjE1lJHCoFZr4UmIlraVRyJailVU44rntnqH1BCxjeSC/Pxfr/iwyN8fRGiRe7wZ MWpemsalT0E6b7fKtpmgBOrr22WoPOulZaCq5PBG3AclMBogKKPb/OXzU8sihVbDAq 6ttokefrni1lA== From: cel@kernel.org To: Cc: Chuck Lever Subject: [RFC PATCH 5/5] guestfs: Convert part of scripts/bringup_guestfs.sh to Ansible Date: Thu, 22 May 2025 09:31:37 -0400 Message-ID: <20250522133137.989457-6-cel@kernel.org> X-Mailer: git-send-email 2.49.0 In-Reply-To: <20250522133137.989457-1-cel@kernel.org> References: <20250522133137.989457-1-cel@kernel.org> Precedence: bulk X-Mailing-List: kdevops@lists.linux.dev List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: 8bit From: Chuck Lever The part of bringup_guestfs.sh that provisions and starts up target nodes is converted to Ansible. This parallelizes node bringup. The new Ansible code takes a stab at being more idempotent than the script was, as well. The part of bringup_guestfs.sh that creates missing base images is left in place for the moment. Signed-off-by: Chuck Lever --- playbooks/roles/guestfs/defaults/main.yml | 1 + playbooks/roles/guestfs/tasks/bringup.yml | 157 ++++++++++++++++++ playbooks/roles/guestfs/tasks/extra_disks.yml | 16 ++ playbooks/roles/guestfs/tasks/largeio.yml | 11 ++ playbooks/roles/guestfs/tasks/main.yml | 6 + scripts/bringup_guestfs.sh | 76 +-------- scripts/guestfs.Makefile | 5 + 7 files changed, 197 insertions(+), 75 deletions(-) create mode 100644 playbooks/roles/guestfs/tasks/bringup.yml create mode 100644 playbooks/roles/guestfs/tasks/extra_disks.yml create mode 100644 playbooks/roles/guestfs/tasks/largeio.yml diff --git a/playbooks/roles/guestfs/defaults/main.yml b/playbooks/roles/guestfs/defaults/main.yml index dc955d915d70..448ff1f0c6c5 100644 --- a/playbooks/roles/guestfs/defaults/main.yml +++ b/playbooks/roles/guestfs/defaults/main.yml @@ -2,3 +2,4 @@ --- libvirt_uri_system: false distro_debian_based: false +libvirt_enable_largeio: false diff --git a/playbooks/roles/guestfs/tasks/bringup.yml b/playbooks/roles/guestfs/tasks/bringup.yml new file mode 100644 index 000000000000..b871e2c82ffa --- /dev/null +++ b/playbooks/roles/guestfs/tasks/bringup.yml @@ -0,0 +1,157 @@ +--- +- name: Check if target nodes are already defined + ansible.builtin.command: + cmd: "virsh domstate {{ inventory_hostname }}" + register: domstate_output + changed_when: false + failed_when: false + +- name: Ensure the target node is up + community.libvirt.virt: + name: "{{ inventory_hostname }}" + uri: "{{ libvirt_uri }}" + state: running + when: + - domstate_output.rc == 0 + +- name: The target node is already defined + ansible.builtin.meta: end_host + when: + - domstate_output.rc == 0 + +- name: Set the pathname of the ssh directory for the target node + ansible.builtin.set_fact: + ssh_key_dir: "{{ guestfs_path }}/{{ inventory_hostname }}/ssh" + +- name: Set the pathname of the ssh key for the target node + ansible.builtin.set_fact: + ssh_key: "{{ ssh_key_dir }}/id_ed25519" + +- name: Generate ssh keys for the target node + block: + - name: Create the ssh key directory on the control host + ansible.builtin.file: + path: "{{ ssh_key_dir }}" + state: directory + mode: "u=rwx" + +# - name: Destroy old keys for the target node +# ansible.builtin.file: +# path: "{{ item }}" +# state: absent +# loop: +# - "{{ ssh_key }}" +# - "{{ ssh_key }}.pub" + + - name: Generate fresh keys for the target node + ansible.builtin.command: + cmd: 'ssh-keygen -q -t ed25519 -f {{ ssh_key }} -N ""' + +- name: Set the pathname of storage pool directory + ansible.builtin.set_fact: + storagedir: "{{ kdevops_storage_pool_path }}/guestfs" + +- name: Set the pathname of root image for the target node + ansible.builtin.set_fact: + rootimg: "{{ storagedir }}/{{ inventory_hostname }}/root.raw" + +- name: Set the pathname of the OS base image + ansible.builtin.set_fact: + base_image: "{{ storagedir }}/base_images/{{ virtbuilder_os_version }}.raw" + +- name: Create the storage pool directory for the target node + ansible.builtin.file: + path: "{{ storagedir }}/{{ inventory_hostname }}" + state: directory + +- name: Copy the base image + ansible.builtin.command: + cmd: "cp --reflink=auto {{ base_image }} {{ rootimg }}" + +- name: Get the timezone of the control host + ansible.builtin.command: + cmd: "timedatectl show -p Timezone --value" + register: host_timezone + +- name: Prep the boot image for the target node (as root) + become: true + become_method: ansible.builtin.sudo + ansible.builtin.command: + argv: + - "virt-sysprep" + - "-a" + - "{{ rootimg }}" + - "--hostname" + - "{{ inventory_hostname }}" + - "--ssh-inject" + - "kdevops:file:{{ ssh_key }}.pub" + - "--timezone" + - "{{ host_timezone.stdout }}" + when: + - libvirt_uri_system|bool + +- name: Prep the boot image for the target node (non-root) + ansible.builtin.command: + argv: + - "virt-sysprep" + - "-a" + - "{{ rootimg }}" + - "--hostname" + - "{{ inventory_hostname }}" + - "--ssh-inject" + - "kdevops:file:{{ ssh_key }}.pub" + - "--timezone" + - "{{ host_timezone.stdout }}" + when: + - not libvirt_uri_system|bool + +- name: Build largeio devices + ansible.builtin.include_tasks: + file: "{{ role_path }}/tasks/largeio.yml" + when: + - libvirt_enable_largeio|bool + +- name: Create extra disks + vars: + path: "{{ storagedir }}/{{ inventory_hostname }}/extra{{ item }}.{{ libvirt_extra_drive_format }}" + ansible.builtin.include_tasks: + file: "{{ role_path }}/tasks/extra_disks.yml" + loop: "{{ range(0, 4) | list }}" + when: + - not libvirt_enable_largeio|bool + +- name: Define the target nodes + vars: + xml_file: "{{ guestfs_path }}/{{ inventory_hostname }}/{{ inventory_hostname }}.xml" + community.libvirt.virt: + command: define + name: "{{ inventory_hostname }}" + xml: "{{ lookup('file', xml_file) }}" + uri: "{{ libvirt_uri }}" + +- name: Find PCIe passthrough devices + ansible.builtin.find: + paths: "{{ guestfs_path }}/{{ inventory_hostname }}" + file_type: file + patterns: "pcie_passthrough_*.xml" + register: passthrough_devices + +- name: Attach PCIe passthrough devices + ansible.builtin.command: + argv: + - "virsh" + - "attach-device" + - "{{ inventory_hostname }}" + - "{{ item }}" + - "--config" + loop: "{{ passthrough_devices.files }}" + loop_control: + label: "Doing PCI-E passthrough for device {{ item }}" + when: + - passthrough_devices.matched > 0 + +- name: Boot the target nodes + community.libvirt.virt: + name: "{{ inventory_hostname }}" + uri: "{{ libvirt_uri }}" + state: running diff --git a/playbooks/roles/guestfs/tasks/extra_disks.yml b/playbooks/roles/guestfs/tasks/extra_disks.yml new file mode 100644 index 000000000000..c8a9bd63885f --- /dev/null +++ b/playbooks/roles/guestfs/tasks/extra_disks.yml @@ -0,0 +1,16 @@ +--- +- name: Create the new drive image + ansible.builtin.command: + argv: + - "qemu-img" + - "create" + - "-f" + - "{{ libvirt_extra_drive_format }}" + - "{{ path }}" + - "100G" + +- name: Adjust the permission settings of the drive image file + ansible.builtin.file: + path: "{{ path }}" + group: "{{ libvirt_qemu_group }}" + mode: "g+rw,o-rw" diff --git a/playbooks/roles/guestfs/tasks/largeio.yml b/playbooks/roles/guestfs/tasks/largeio.yml new file mode 100644 index 000000000000..4246677d18d8 --- /dev/null +++ b/playbooks/roles/guestfs/tasks/largeio.yml @@ -0,0 +1,11 @@ +--- +- name: Compute the total number of devices to build + ansible.builtin.set_fact: + total_devices: "{{ libvirt_largeio_pow_limit * libvirt_largeio_drives_per_space }}" + +- name: Create largeio block devices + ansible.builtin.include_tasks: + file: "{{ role_path }}/tasks/extra_disks.yml" + vars: + path: "{{ storagedir }}/{{ inventory_hostname }}/extra{{ item }}.{{ libvirt_extra_drive_format }}" + loop: "{{ range(0, total_devices) | list }}" diff --git a/playbooks/roles/guestfs/tasks/main.yml b/playbooks/roles/guestfs/tasks/main.yml index bda91de79983..e63484b0229d 100644 --- a/playbooks/roles/guestfs/tasks/main.yml +++ b/playbooks/roles/guestfs/tasks/main.yml @@ -23,6 +23,12 @@ ansible.builtin.import_tasks: file: "{{role_path }}/tasks/network.yml" +- name: Bring up each target node + tags: + - bringup + ansible.builtin.import_tasks: + file: "{{role_path }}/tasks/bringup.yml" + - name: Set up target node console permissions tags: - console-permissions diff --git a/scripts/bringup_guestfs.sh b/scripts/bringup_guestfs.sh index 67f85a5fdb0a..be9ec3405037 100755 --- a/scripts/bringup_guestfs.sh +++ b/scripts/bringup_guestfs.sh @@ -322,78 +322,4 @@ if [ ! -f $BASE_IMAGE ]; then fi fi -# FIXME: is there a yaml equivalent of jq? -grep -e '^ - name: ' ${TOPDIR}/guestfs/kdevops_nodes.yaml | sed 's/^ - name: //' | while read name -do - # - # If the guest is already defined, then just stop what we're doing - # and plead to the developer to clean things up. - # - if virsh list --all | grep --quiet --word-regexp "$name"; then - output_domstate=$(virsh domstate $name 2>/dev/null) - echo "Domain $name is already defined. (state: $output_domstate)" - if [ "$output_domstate" != "running" ]; then - virsh start $name - fi - exit 0 - fi - - SSH_KEY_DIR="${GUESTFSDIR}/$name/ssh" - SSH_KEY="${SSH_KEY_DIR}/id_ed25519" - - # Generate a new ssh key - mkdir -p "$SSH_KEY_DIR" - chmod 0700 "$SSH_KEY_DIR" - rm -f $SSH_KEY $SSH_KEY.pub - ssh-keygen -q -t ed25519 -f $SSH_KEY -N "" - - mkdir -p "$STORAGEDIR/$name" - - # Copy the base image and prep it - ROOTIMG="$STORAGEDIR/$name/root.raw" - cp --reflink=auto $BASE_IMAGE $ROOTIMG - TZ="$(timedatectl show -p Timezone --value)" - $USE_SUDO virt-sysprep -a $ROOTIMG --hostname $name --ssh-inject "kdevops:file:$SSH_KEY.pub" --timezone $TZ - - if [[ "${CONFIG_LIBVIRT_ENABLE_LARGEIO+x}" && \ - "$CONFIG_LIBVIRT_ENABLE_LARGEIO" == "y" ]]; then - lbs_idx=0 - for i in $(seq 1 $(($CONFIG_QEMU_LARGEIO_MAX_POW_LIMIT+1))); do - for x in $(seq 0 $CONFIG_QEMU_EXTRA_DRIVE_LARGEIO_NUM_DRIVES_PER_SPACE); do - diskimg="$STORAGEDIR/$name/extra${lbs_idx}.${IMG_FMT}" - rm -f $diskimg - qemu-img create -f $IMG_FMT "$diskimg" 100G - if [[ "$CONFIG_LIBVIRT_URI_SYSTEM" == "y" ]]; then - chmod g+rw $diskimg - chgrp $QEMU_GROUP $diskimg - fi - let lbs_idx=$lbs_idx+1 - done - done - else - # build some extra disks - for i in $(seq 0 3); do - diskimg="$STORAGEDIR/$name/extra${i}.${IMG_FMT}" - rm -f $diskimg - qemu-img create -f $IMG_FMT "$STORAGEDIR/$name/extra${i}.$IMG_FMT" 100G - if [[ "$CONFIG_LIBVIRT_URI_SYSTEM" == "y" ]]; then - chmod g+rw $STORAGEDIR/$name/extra${i}.$IMG_FMT - chgrp $QEMU_GROUP $STORAGEDIR/$name/extra${i}.$IMG_FMT - fi - done - fi - - virsh define $GUESTFSDIR/$name/$name.xml - XML_DEVICES_COUNT=$(find $GUESTFSDIR/$name/ -name pcie_passthrough_*.xml | wc -l) - if [[ $XML_DEVICES_COUNT -gt 0 ]]; then - for xml in $GUESTFSDIR/$name/pcie_passthrough_*.xml; do - echo "Doing PCI-E passthrough for device $xml" - virsh attach-device $name $xml --config - done - fi - virsh start $name - if [[ $? -ne 0 ]]; then - echo "Failed to start $name" - exit 1 - fi -done +exit 0 diff --git a/scripts/guestfs.Makefile b/scripts/guestfs.Makefile index 290315ee9c9e..84cf99db982d 100644 --- a/scripts/guestfs.Makefile +++ b/scripts/guestfs.Makefile @@ -81,6 +81,11 @@ bringup_guestfs: $(GUESTFS_BRINGUP_DEPS) --extra-vars=@./extra_vars.yaml \ --tags config-check,network,storage-pool-path $(Q)$(TOPDIR)/scripts/bringup_guestfs.sh + $(Q)ansible-playbook $(ANSIBLE_VERBOSE) \ + -i hosts \ + playbooks/guestfs.yml \ + --extra-vars=@./extra_vars.yaml \ + --tags bringup $(Q)ansible-playbook $(ANSIBLE_VERBOSE) \ --inventory localhost, \ playbooks/guestfs.yml \ -- 2.49.0