[PATCH 06/10] guestfs: add support to infer host distro mirrororing optimizations

[Luis Chamberlain <mcgrof@kernel.org>]

For production level uses of kdevops you want to optimize every ounce of
time, and network bandwidth. kdevops already mirrors everything we can
git clone, the next optimization is to ensure both the host and guest
use their own local mirrors for packaging. This is resolved by the
host and we don't care how, but *if* we can detect this optimization
in place, *and* if the host and guest match distros, then it makes
sense to leverage the host's sources file for the repo for the guest.

So, this optimization requires no user input. We start with debian
support, and add enough kconfig magic to let others easily add support
for other distros, all you need is the mapping file and how to detect
when you think it is safe from a host perspective for a guest to use
your sources file.

The first heuristic is a simple traceroute to check how many hops
your host uses for its sources, if its 1 chances are pretty high
the host is using a mirror for its packaging sources. This is also
distros specific as each source file differs, for debian its
/etc/apt/sources.list, and so we traceroute the first entry. Second
heuristic is determining when we can use the host sources on the
guest. For debian we enable this for now when the host is trixie and
the guest is either trixie or buster. That is, if the host is using
trixie we'll upgrade a buster guest to trixie.

We've always wanted trixie for folks working on hacking, its just we
can't use it yet...

If you only want buster, then your host probably only wants to use
buster too.

Signed-off-by: Luis Chamberlain <mcgrof@kernel.org>
---
 kconfigs/Kconfig.distro                     |  4 ++
 kconfigs/Kconfig.guestfs                    | 47 +++++++++++++++++++++
 scripts/bringup_guestfs.sh                  | 13 ++++++
 scripts/get-distro-has-hop-count-sources.sh | 39 +++++++++++++++++
 4 files changed, 103 insertions(+)
 create mode 100755 scripts/get-distro-has-hop-count-sources.sh

diff --git a/kconfigs/Kconfig.distro b/kconfigs/Kconfig.distro
index 18e582eda40b..8289a3ab89e1 100644
--- a/kconfigs/Kconfig.distro
+++ b/kconfigs/Kconfig.distro
@@ -12,6 +12,10 @@ config DISTRO_DEBIAN_TRIXIE
 	bool
 	default $(shell, scripts/os-debian-version.sh trixie)
 
+config DISTRO_DEBIAN_HAS_HOP1_SOURCES
+	bool
+	default $(shell, scripts/get-distro-has-hop-count-sources.sh 1)
+
 endif
 
 config DISTRO_FEDORA
diff --git a/kconfigs/Kconfig.guestfs b/kconfigs/Kconfig.guestfs
index 5839fbedfd08..dd0b280edbe5 100644
--- a/kconfigs/Kconfig.guestfs
+++ b/kconfigs/Kconfig.guestfs
@@ -25,6 +25,21 @@ config GUESTFS_CUSTOM_RAW_IMAGE_SHA512SUMS_URL
 config GUESTFS_HAS_CUSTOM_RAW_IMAGE_ROLLING
 	bool
 
+config GUESTFS_HAS_DISTRO_SOURCE_COPY_CAPABILITIES
+	bool
+
+if GUESTFS_HAS_DISTRO_SOURCE_COPY_CAPABILITIES
+
+config GUESTFS_COPY_SOURCES_FROM_HOST_TO_GUEST
+	bool
+
+config GUESTFS_DISTRO_SOURCE_AND_DEST_FILE
+	string
+	depends on GUESTFS_COPY_SOURCES_FROM_HOST_TO_GUEST
+	default "/etc/apt/sources.list" if GUESTFS_DEBIAN
+
+endif
+
 choice
 	prompt "Guestfs Linux distribution to use"
 	default GUESTFS_FEDORA if DISTRO_FEDORA || DISTRO_REDHAT
@@ -51,6 +66,7 @@ config GUESTFS_DEBIAN
 	select HAVE_DISTRO_PREFERS_CUSTOM_HOST_PREFIX
 	select HAVE_DISTRO_PREFERS_FSTESTS_WATCHDOG if KDEVOPS_WORKFLOW_ENABLE_FSTESTS
 	select HAVE_DISTRO_PREFERS_FSTESTS_WATCHDOG_KILL if KDEVOPS_WORKFLOW_ENABLE_FSTESTS
+	select GUESTFS_HAS_DISTRO_SOURCE_COPY_CAPABILITIES
 	help
 	  This option will set the target guest to Debian.
 
@@ -115,6 +131,15 @@ endchoice
 
 endif # GUESTFS_DEBIAN_TRIXIE
 
+config GUESTFS_DEBIAN_HOST_UPDATES_GUEST
+	bool
+	default y if GUESTFS_DEBIAN_BUSTER && DISTRO_DEBIAN_TRIXIE
+
+config GUESTFS_DEBIAN_HOST_GUEST_MATCH
+	bool
+	default y if GUESTFS_DEBIAN_BUSTER && DISTRO_DEBIAN_BUSTER
+	default y if GUESTFS_DEBIAN_TRIXIE && DISTRO_DEBIAN_TRIXIE
+
 config GUESTFS_DEBIAN_IMAGE_NAME
 	string
 	default "debian-13-generic-amd64-daily" if GUESTFS_DEBIAN_TRIXIE_GENERIC_AMD64
@@ -127,6 +152,28 @@ config GUESTFS_DEBIAN_BOX_SHORT
         default "debian13" if GUESTFS_DEBIAN_TRIXIE
         default "debian12" if GUESTFS_DEBIAN_BUSTER
 
+config GUESTFS_DEBIAN_COPY_HOST_SOURCES
+	bool
+	depends on GUESTFS_DEBIAN_HOST_GUEST_MATCH || GUESTFS_DEBIAN_HOST_UPDATES_GUEST
+	default DISTRO_DEBIAN_HAS_HOP1_SOURCES
+	select GUESTFS_COPY_SOURCES_FROM_HOST_TO_GUEST
+	help
+	  We have detected that you are running debian on the host, this option
+	  will be enabled by default if we detect that your /etc/apt/sources.list
+	  is using a source we determine has only one hop via traceroute to it,
+	  for details of that heuristic see the script:
+
+	    ./scripts/get-distro-has-hop-count-sources.sh
+
+	  If your hop distance to your mirror is just 1, you are using debian on
+	  the host, want to use debian guests, both host and guest match the
+	  target release, then its a good assumtion you'd likely want to take
+	  advantage of that same mirror for your guests. So disable this if you
+	  really don't want to take advantage of your sources.list.
+
+	  If this is disabled but you are sure you can use your host's
+	  /etc/apt/sources.list on the guests then enable this.
+
 endif # GUESTFS_DEBIAN
 
 config VIRT_BUILDER_OS_VERSION
diff --git a/scripts/bringup_guestfs.sh b/scripts/bringup_guestfs.sh
index 514a26a60436..c410d0cf725d 100755
--- a/scripts/bringup_guestfs.sh
+++ b/scripts/bringup_guestfs.sh
@@ -156,6 +156,15 @@ build_custom_image()
 	# just build $virt-builder, which is the pristine upstream image.
 }
 
+copy_host_sources()
+{
+	TARGET_DIR="$(dirname $CONFIG_GUESTFS_DISTRO_SOURCE_AND_DEST_FILE)"
+	cat <<_EOT >>$cmdfile
+mkdir $TARGET_DIR
+copy-in $CONFIG_GUESTFS_DISTRO_SOURCE_AND_DEST_FILE:$TARGET_DIR
+_EOT
+}
+
 mkdir -p $STORAGEDIR
 mkdir -p $BASE_IMAGE_DIR
 
@@ -188,6 +197,10 @@ copy-in $CONFIG_KDEVOPS_CUSTOM_YUM_REPOFILE:/etc/yum.repos.d
 _EOT
 	fi
 
+	if [[ "$CONFIG_GUESTFS_COPY_SOURCES_FROM_HOST_TO_GUEST" == "y" ]]; then
+		copy_host_sources
+	fi
+
 # basic pre-install customization
 	cat <<_EOT >>$cmdfile
 install sudo,qemu-guest-agent,python3,bash
diff --git a/scripts/get-distro-has-hop-count-sources.sh b/scripts/get-distro-has-hop-count-sources.sh
new file mode 100755
index 000000000000..1db975468887
--- /dev/null
+++ b/scripts/get-distro-has-hop-count-sources.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+DEBIAN_VERSION_FILE="/etc/debian_version"
+ACCEPTABLE_HOPS=1
+
+if [[ $# -eq 1 ]]; then
+	ACCEPTABLE_HOPS=$1
+fi
+
+# For now we only support debian. Adding other distros should be easy, its
+# just a matter of mapping a sources file to a column we can use for a server
+# hostname.
+if [[ ! -f $DEBIAN_VERSION_FILE ]]; then
+	echo n
+	exit 0
+fi
+
+SOURCES_FILE="/etc/apt/sources.list"
+
+if [[ ! -f $SOURCES_FILE ]]; then
+	echo n
+	exit 0
+fi
+
+which traceroute > /dev/null
+if [[ $? -ne 0 ]]; then
+	echo n
+	exit 0
+fi
+
+LINE=$(grep -v "^#" $SOURCES_FILE | head -1)
+HOST_URL_LINE=$(echo $LINE | awk '{print $2}')
+HOST=$(echo $HOST_URL_LINE | awk -F[/:] '{print $4}')
+HOP_COUNT=$(traceroute -n -w 1,1,1 $HOST | wc -l)
+HOP_COUNT=$((HOP_COUNT -1))
+
+if [[ $HOP_COUNT -le $ACCEPTABLE_HOPS ]]; then
+	echo y
+	exit 0
+fi
-- 
2.43.0