From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from bombadil.infradead.org (bombadil.infradead.org [198.137.202.133]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id AAEDC208D1 for ; Wed, 8 May 2024 06:50:40 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=198.137.202.133 ARC-Seal:i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1715151042; cv=none; b=MtY6IqhggX03mLu69gsppxIfi7izHoNsLZfnrq0c/2HMfxG+Thqn5IisIcvRifB9QvMQO/4Qyr/6u8qZN98TWYoVZCb3snWtXWe8AxF0rmXm+kRjB9cXDfBaWtpsX9LkIi9NrAJZoaN8ecpL22kc69hKTEZDO2LubYl1XyfFrIw= ARC-Message-Signature:i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1715151042; c=relaxed/simple; bh=qtahONTLtW81C08LRP0sQDjnCYQIh5pMN0CdTwGHfIc=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=Jb7SPWl6OAsgE9JKjtwBOkFqzkUU4R6XXf1FkSA/h8ufFYWbe71JWX92p1LQevuAxWtxYTNQqAzx1ch6j0L6WGMDzQWs0mvcYRX09RIZl5e+onz5Y/y7U3r8QxVO1C/O1hbEgGmnzmhPizTekqZp1aM9bxj6Yf5rmlWUd2GbCeg= ARC-Authentication-Results:i=1; smtp.subspace.kernel.org; dmarc=fail (p=none dis=none) header.from=kernel.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=qmQtFKNh; arc=none smtp.client-ip=198.137.202.133 Authentication-Results: smtp.subspace.kernel.org; dmarc=fail (p=none dis=none) header.from=kernel.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="qmQtFKNh" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=bombadil.20210309; h=Sender:Content-Transfer-Encoding: MIME-Version:References:In-Reply-To:Message-ID:Date:Subject:Cc:To:From: Reply-To:Content-Type:Content-ID:Content-Description; bh=3bOOnFPuB+bYM9SE+hGBd7i4ifv9pYj1KJ4ep43qC/M=; b=qmQtFKNhNCVGZfyVOWg2qtmZBG Y5hLmL6tlA3EBZYB3mBqJRqC7Fs62nIwHCuiqr9qxQT6pbLqWcZRTyb4k3UD+M+mUBs3e4CluWEVz Xnm6RX/YVNgSJ5d32301815DP5Pa5bJtLAKtpYcXbReZCOb0Lo3MICGnIKI1nSZRpolqapjycGgOL 0t4+0jU93416pk5xFs0SLHlLK9cCWe6nmZGb5FkX7nmQY/r63HMfkUUknfov0VKrieFd0M1FrpHay v1Fy5kn9Frf619xsaeltMwLfXCKIUNIyEUNQsCVao/IQTqZd97uB08HcX9icgPg6OPO5t6UP60wCs +ED8xZlg==; Received: from mcgrof by bombadil.infradead.org with local (Exim 4.97.1 #2 (Red Hat Linux)) id 1s4b8S-0000000EIkW-10ew; Wed, 08 May 2024 06:50:40 +0000 From: Luis Chamberlain To: kdevops@lists.linux.dev Cc: mcgrof@kernel.org Subject: [PATCH 06/10] guestfs: add support to infer host distro mirrororing optimizations Date: Tue, 7 May 2024 23:50:34 -0700 Message-ID: <20240508065039.3408637-7-mcgrof@kernel.org> X-Mailer: git-send-email 2.44.0 In-Reply-To: <20240508065039.3408637-1-mcgrof@kernel.org> References: <20240508065039.3408637-1-mcgrof@kernel.org> Precedence: bulk X-Mailing-List: kdevops@lists.linux.dev List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Sender: Luis Chamberlain For production level uses of kdevops you want to optimize every ounce of time, and network bandwidth. kdevops already mirrors everything we can git clone, the next optimization is to ensure both the host and guest use their own local mirrors for packaging. This is resolved by the host and we don't care how, but *if* we can detect this optimization in place, *and* if the host and guest match distros, then it makes sense to leverage the host's sources file for the repo for the guest. So, this optimization requires no user input. We start with debian support, and add enough kconfig magic to let others easily add support for other distros, all you need is the mapping file and how to detect when you think it is safe from a host perspective for a guest to use your sources file. The first heuristic is a simple traceroute to check how many hops your host uses for its sources, if its 1 chances are pretty high the host is using a mirror for its packaging sources. This is also distros specific as each source file differs, for debian its /etc/apt/sources.list, and so we traceroute the first entry. Second heuristic is determining when we can use the host sources on the guest. For debian we enable this for now when the host is trixie and the guest is either trixie or buster. That is, if the host is using trixie we'll upgrade a buster guest to trixie. We've always wanted trixie for folks working on hacking, its just we can't use it yet... If you only want buster, then your host probably only wants to use buster too. Signed-off-by: Luis Chamberlain --- kconfigs/Kconfig.distro | 4 ++ kconfigs/Kconfig.guestfs | 47 +++++++++++++++++++++ scripts/bringup_guestfs.sh | 13 ++++++ scripts/get-distro-has-hop-count-sources.sh | 39 +++++++++++++++++ 4 files changed, 103 insertions(+) create mode 100755 scripts/get-distro-has-hop-count-sources.sh diff --git a/kconfigs/Kconfig.distro b/kconfigs/Kconfig.distro index 18e582eda40b..8289a3ab89e1 100644 --- a/kconfigs/Kconfig.distro +++ b/kconfigs/Kconfig.distro @@ -12,6 +12,10 @@ config DISTRO_DEBIAN_TRIXIE bool default $(shell, scripts/os-debian-version.sh trixie) +config DISTRO_DEBIAN_HAS_HOP1_SOURCES + bool + default $(shell, scripts/get-distro-has-hop-count-sources.sh 1) + endif config DISTRO_FEDORA diff --git a/kconfigs/Kconfig.guestfs b/kconfigs/Kconfig.guestfs index 5839fbedfd08..dd0b280edbe5 100644 --- a/kconfigs/Kconfig.guestfs +++ b/kconfigs/Kconfig.guestfs @@ -25,6 +25,21 @@ config GUESTFS_CUSTOM_RAW_IMAGE_SHA512SUMS_URL config GUESTFS_HAS_CUSTOM_RAW_IMAGE_ROLLING bool +config GUESTFS_HAS_DISTRO_SOURCE_COPY_CAPABILITIES + bool + +if GUESTFS_HAS_DISTRO_SOURCE_COPY_CAPABILITIES + +config GUESTFS_COPY_SOURCES_FROM_HOST_TO_GUEST + bool + +config GUESTFS_DISTRO_SOURCE_AND_DEST_FILE + string + depends on GUESTFS_COPY_SOURCES_FROM_HOST_TO_GUEST + default "/etc/apt/sources.list" if GUESTFS_DEBIAN + +endif + choice prompt "Guestfs Linux distribution to use" default GUESTFS_FEDORA if DISTRO_FEDORA || DISTRO_REDHAT @@ -51,6 +66,7 @@ config GUESTFS_DEBIAN select HAVE_DISTRO_PREFERS_CUSTOM_HOST_PREFIX select HAVE_DISTRO_PREFERS_FSTESTS_WATCHDOG if KDEVOPS_WORKFLOW_ENABLE_FSTESTS select HAVE_DISTRO_PREFERS_FSTESTS_WATCHDOG_KILL if KDEVOPS_WORKFLOW_ENABLE_FSTESTS + select GUESTFS_HAS_DISTRO_SOURCE_COPY_CAPABILITIES help This option will set the target guest to Debian. @@ -115,6 +131,15 @@ endchoice endif # GUESTFS_DEBIAN_TRIXIE +config GUESTFS_DEBIAN_HOST_UPDATES_GUEST + bool + default y if GUESTFS_DEBIAN_BUSTER && DISTRO_DEBIAN_TRIXIE + +config GUESTFS_DEBIAN_HOST_GUEST_MATCH + bool + default y if GUESTFS_DEBIAN_BUSTER && DISTRO_DEBIAN_BUSTER + default y if GUESTFS_DEBIAN_TRIXIE && DISTRO_DEBIAN_TRIXIE + config GUESTFS_DEBIAN_IMAGE_NAME string default "debian-13-generic-amd64-daily" if GUESTFS_DEBIAN_TRIXIE_GENERIC_AMD64 @@ -127,6 +152,28 @@ config GUESTFS_DEBIAN_BOX_SHORT default "debian13" if GUESTFS_DEBIAN_TRIXIE default "debian12" if GUESTFS_DEBIAN_BUSTER +config GUESTFS_DEBIAN_COPY_HOST_SOURCES + bool + depends on GUESTFS_DEBIAN_HOST_GUEST_MATCH || GUESTFS_DEBIAN_HOST_UPDATES_GUEST + default DISTRO_DEBIAN_HAS_HOP1_SOURCES + select GUESTFS_COPY_SOURCES_FROM_HOST_TO_GUEST + help + We have detected that you are running debian on the host, this option + will be enabled by default if we detect that your /etc/apt/sources.list + is using a source we determine has only one hop via traceroute to it, + for details of that heuristic see the script: + + ./scripts/get-distro-has-hop-count-sources.sh + + If your hop distance to your mirror is just 1, you are using debian on + the host, want to use debian guests, both host and guest match the + target release, then its a good assumtion you'd likely want to take + advantage of that same mirror for your guests. So disable this if you + really don't want to take advantage of your sources.list. + + If this is disabled but you are sure you can use your host's + /etc/apt/sources.list on the guests then enable this. + endif # GUESTFS_DEBIAN config VIRT_BUILDER_OS_VERSION diff --git a/scripts/bringup_guestfs.sh b/scripts/bringup_guestfs.sh index 514a26a60436..c410d0cf725d 100755 --- a/scripts/bringup_guestfs.sh +++ b/scripts/bringup_guestfs.sh @@ -156,6 +156,15 @@ build_custom_image() # just build $virt-builder, which is the pristine upstream image. } +copy_host_sources() +{ + TARGET_DIR="$(dirname $CONFIG_GUESTFS_DISTRO_SOURCE_AND_DEST_FILE)" + cat <<_EOT >>$cmdfile +mkdir $TARGET_DIR +copy-in $CONFIG_GUESTFS_DISTRO_SOURCE_AND_DEST_FILE:$TARGET_DIR +_EOT +} + mkdir -p $STORAGEDIR mkdir -p $BASE_IMAGE_DIR @@ -188,6 +197,10 @@ copy-in $CONFIG_KDEVOPS_CUSTOM_YUM_REPOFILE:/etc/yum.repos.d _EOT fi + if [[ "$CONFIG_GUESTFS_COPY_SOURCES_FROM_HOST_TO_GUEST" == "y" ]]; then + copy_host_sources + fi + # basic pre-install customization cat <<_EOT >>$cmdfile install sudo,qemu-guest-agent,python3,bash diff --git a/scripts/get-distro-has-hop-count-sources.sh b/scripts/get-distro-has-hop-count-sources.sh new file mode 100755 index 000000000000..1db975468887 --- /dev/null +++ b/scripts/get-distro-has-hop-count-sources.sh @@ -0,0 +1,39 @@ +#!/bin/bash +DEBIAN_VERSION_FILE="/etc/debian_version" +ACCEPTABLE_HOPS=1 + +if [[ $# -eq 1 ]]; then + ACCEPTABLE_HOPS=$1 +fi + +# For now we only support debian. Adding other distros should be easy, its +# just a matter of mapping a sources file to a column we can use for a server +# hostname. +if [[ ! -f $DEBIAN_VERSION_FILE ]]; then + echo n + exit 0 +fi + +SOURCES_FILE="/etc/apt/sources.list" + +if [[ ! -f $SOURCES_FILE ]]; then + echo n + exit 0 +fi + +which traceroute > /dev/null +if [[ $? -ne 0 ]]; then + echo n + exit 0 +fi + +LINE=$(grep -v "^#" $SOURCES_FILE | head -1) +HOST_URL_LINE=$(echo $LINE | awk '{print $2}') +HOST=$(echo $HOST_URL_LINE | awk -F[/:] '{print $4}') +HOP_COUNT=$(traceroute -n -w 1,1,1 $HOST | wc -l) +HOP_COUNT=$((HOP_COUNT -1)) + +if [[ $HOP_COUNT -le $ACCEPTABLE_HOPS ]]; then + echo y + exit 0 +fi -- 2.43.0