[PATCH OSSTEST v8 05/14] distros: Support pvgrub for Wheezy too.

[Ian Campbell <ian.campbell@citrix.com>]

This requires us to install pv-grub-menu from backports, which we do
using a late_command.

Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
---
v8:
  - Use a heredoc for sources.list additions, since this was suggested
    by Ian I would have retained the ack apart from the second change.
  - Dropped unused $cd argument to setup_netinst. (Noticed when I came
    to document <gident>_cd and found it wasn't used!)
v7:
  - Remove vestigial attempts to enable -backports via d-i preseeding.
v3:
  - Remove spurious () from <<(END) (and the prexisting "" too)
  - Remove $xopts{EnableBackports} and automatically handle the need to add
    backports in preseed_base.
  - Install via late_command not apt-setup, since the former has
    issues, hence subject drops "attempt to..."
---
 Osstest/Debian.pm    | 40 ++++++++++++++++++++++++---
 make-distros-flight  | 58 +++++++++++++++++++++++++++++++++++++--
 ts-debian-di-install | 76 +++++++++++++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 168 insertions(+), 6 deletions(-)

diff --git a/Osstest/Debian.pm b/Osstest/Debian.pm
index 1edf49f..7c94b6c 100644
--- a/Osstest/Debian.pm
+++ b/Osstest/Debian.pm
@@ -777,8 +777,6 @@ sub preseed_base ($$$$;@) {
     preseed_hook_overlay($ho, $sfx, 'overlay', 'overlay.tar');
 
     my $preseed = <<"END";
-d-i mirror/suite string $suite
-
 d-i debian-installer/locale string en_GB
 d-i console-keymaps-at/keymap select gb
 d-i keyboard-configuration/xkb-keymap string en_GB
@@ -854,6 +852,11 @@ END
 d-i clock-setup/ntp-server string $ntpserver
 END
 
+    # For CDROM the suite is part of the image
+    $preseed .= <<END unless $xopts{CDROM};
+d-i mirror/suite string $suite
+END
+
     $preseed .= <<"END";
 
 ### END OF DEBIAN PRESEED BASE
@@ -867,7 +870,38 @@ sub preseed_create_guest ($$;@) {
 
     my $suite= $xopts{Suite} || $c{DebianSuite};
 
-    my $extra_packages = "pv-grub-menu" if $xopts{PvMenuLst};
+    my $extra_packages = "";
+    if ($xopts{PvMenuLst}) {
+        if ($suite =~ m/wheezy/) {
+            # pv-grub-menu/wheezy-backports + using apt-setup to add
+            # backports results in iproute, ifupdown and
+            # isc-dhcp-client getting removed because tasksel's
+            # invocation of apt-get install somehow decides the
+            # iproute2 from wheezy-backports is a thing it wants to
+            # install. So instead lets fake it with a late command...
+            #
+            # This also has the bonus of working round an issue with
+            # 1.2.1~bpo70+1 which created an invalid menu.lst using
+            # "root(/dev/xvda,0)" which pvgrub cannot parse because
+            # the Grub device.map isn't present at pkgsel/include time
+            # but it is by late_command time. This was fixed by
+            # version 1.3 which is in Jessie onwards.
+            preseed_hook_command($ho, 'late_command', $sfx, <<END);
+#!/bin/sh
+set -ex
+
+cat <<EOF >>/target/etc/apt/sources.list
+
+\# $suite backports
+deb http://$c{DebianMirrorHost}/$c{DebianMirrorSubpath} $suite-backports main
+EOF
+in-target apt-get update
+in-target apt-get install -y -t wheezy-backports pv-grub-menu
+END
+        } else {
+            $extra_packages = "pv-grub-menu";
+        }
+    }
 
     my $preseed_file= preseed_base($ho, $suite, $sfx, $extra_packages, %xopts);
     $preseed_file.= (<<END);
diff --git a/make-distros-flight b/make-distros-flight
index c19e3ba..49f4b60 100755
--- a/make-distros-flight
+++ b/make-distros-flight
@@ -106,9 +106,9 @@ test_do_one_netboot () {
     arm*_arm*_*) bootloader="pygrub";; # no pvgrub for arm
 
     # Needs a menu.lst, not present in Squeeze+ due to switch to grub2,
-    # workedaround in Jessie+ with pv-grub-menu package.
+    # workedaround in Wheezy+ with pv-grub-menu package (backports in Wheezy,
+    # in Jessie+ main).
     *_squeeze) bootloader="pygrub";;
-    *_wheezy) bootloader="pygrub";;
 
     # pv-grub-x86_64.gz is not built by 32-bit dom0 userspace build.
     i386_amd64_*) bootloader="pygrub";;
@@ -127,6 +127,48 @@ test_do_one_netboot () {
       all_hostflags=$most_hostflags
 }
 
+test_do_one_netinst () {
+  local path_arch
+  case $domU in
+    amd64|i386) path_arch="multi-arch"; file_arch="amd64-i386";;
+    *)          path_arch="$domU";      file_arch="$domU";;
+  esac
+  case $domU in
+    amd64) iso_path="/install.amd/xen";;
+    i386)  iso_path="/install.386/xen";;
+    *)     iso_path="/install.$domU";;
+  esac
+
+  local cdurl=
+  case $cd in
+    current)
+      cdurl="http://cdimage.debian.org/debian-cd/current/${path_arch}/jigdo-cd"
+      ;;
+    weekly)
+      cdurl="http://cdimage.debian.org/cdimage/weekly-builds/${path_arch}/jigdo-cd"
+      ;;
+    *)
+      echo "cd $cd?"
+      exit 1
+      ;;
+  esac
+
+  # Always pygrub since no pv-grub-menu on CD
+  job_create_test                                               \
+   test-$xenarch$kern-$dom0arch-$domU-$cd-netinst-pygrub        \
+    test-debian-di xl $xenarch $dom0arch                        \
+      kernbuildjob=${bfi}build-$dom0arch-$kernbuild             \
+      debian_arch=$domU                                         \
+      debian_method=netinst                                     \
+      debian_netinst_baseurl=$cdurl                             \
+      debian_netinst_filere="debian-.*-${file_arch}-netinst"    \
+      debian_netinst_kernel="${iso_path}/vmlinuz"               \
+      debian_netinst_ramdisk="${iso_path}/initrd.gz"            \
+      debian_bootloader=pygrub                                  \
+      all_hostflags=$most_hostflags
+
+}
+
 test_matrix_do_one () {
   case ${xenarch} in
   amd64) domUarches="amd64 i386";;
@@ -142,6 +184,18 @@ test_matrix_do_one () {
 
     fi
 
+    if [ $debian_suite = "snapshot" ]; then
+        gsuite=sid
+        for cd in current weekly ; do
+          case ${domU}_${gsuite} in
+            armhf_*) continue;; # No iso targets for armhf
+            *) ;;
+          esac
+
+          test_do_one_netinst
+
+        done
+    fi
   done
 }
 
diff --git a/ts-debian-di-install b/ts-debian-di-install
index a59194a..1a7e1d0 100755
--- a/ts-debian-di-install
+++ b/ts-debian-di-install
@@ -21,7 +21,7 @@
 # - <gident>_arch:
 #      Debian arch to install.
 # - <gident>_method:
-#      Install method, currently only "netboot".
+#      Install method, one of "netboot" or "netinst".
 # - <gident>_bootloader:
 #      The PV bootloader to use when booting the guest. One of
 #      "pvgrub" or "pygrub". Default is "pygrub".
@@ -36,6 +36,18 @@
 #    If neither kernel nor ramdisk are specified then the current
 #    TftpDiVersion of d-i will be used, and the runvars will be set to
 #    the file path used.
+#
+# For method="netinst"
+#
+#  - <gident>_netinst_baseurl:
+#       Base URL of directory containing the netinst jigdo images.
+#  - <gident>_netinst_filere:
+#        Regular expression to match actual file within baseurl to use
+#        (without .jigdo suffix)
+#  - <gident>_netinst_kernel:
+#       Path to kernel within the netinst image.
+#  - <gident>_netinst_ramdisk
+#       Path to ramdisk within the netinst image.
 
 use strict qw(vars);
 use DBI;
@@ -68,6 +80,60 @@ sub prep () {
     target_cmd_root($ho, "umount $gho->{Lvdev} ||:");
 }
 
+sub setup_netinst($$)
+{
+    my ($didir, $arch) = @_;
+
+    target_install_packages($ho, qw(jigdo-file));
+
+    my $baseurl = $r{ "$gho->{Guest}_netinst_baseurl" };
+    my $filere = $r{ "$gho->{Guest}_netinst_filere" };
+    my $kernel = $r{ "$gho->{Guest}_netinst_kernel" };
+    my $ramdisk = $r{ "$gho->{Guest}_netinst_ramdisk" };
+
+    die "params" unless $baseurl && $filere && $kernel && $ramdisk;
+
+    my $filebase;
+
+    # Use the MD5SUMs file as an index
+    logm("Fetch index from $baseurl/MD5SUMS");
+    my $idx = target_cmd_output_root($ho, "wget --quiet -O - $baseurl/MD5SUMS");
+    foreach (split /\n/, $idx) {
+	m/^[0-9a-f]{32}  ($filere)\.iso$/ or next;
+	$filebase = $1;
+	last;
+    }
+
+    die unless $filebase;
+
+    logm("Downloading $baseurl/$filebase.jigdo");
+    my $netinst_jigdo = "$baseurl/$filebase.jigdo";
+    # Jigdo uses wget internally, and so obeys $http_proxy. This seems
+    # simpler than the advice at
+    # https://www.debian.org/CD/jigdo-cd/#faq which is to edit
+    # ~/.jigdo-lite and change the wgetOpts setting.
+    my $useproxy = $c{DebianMirrorProxy} // $c{HttpProxy} // '';
+    my $cmd = '';
+    $cmd .= <<END if $useproxy;
+        export http_proxy=$useproxy
+END
+    $cmd .= <<END;
+        cd $didir && jigdo-lite --noask $netinst_jigdo
+END
+    # Jigdo seems to use /etc/apt/sources.list or something, so this
+    # just works using the already configured mirror without
+    # additional configuration, which is good because there doesn't
+    # seem to be any support for such things, at least in Squeeze.
+    target_cmd_root($ho, $cmd, 3600);
+    store_runvar("$gho->{Guest}_netinst_jigdo", $netinst_jigdo);
+
+    return (<<END, "\"file:$didir/$filebase.iso,xvdd:cdrom,r\",");
+bootloader = "pygrub"
+bootloader_args = ["--kernel=$kernel",
+                   "--ramdisk=$ramdisk"]
+END
+}
+
 sub setup_netboot($$$)
 {
     my ($didir, $arch, $suite) = @_;
@@ -130,6 +196,14 @@ END
 
 	$extra_disk = "";
     }
+    elsif ($method eq "netinst" )
+    {
+	logm("$method $arch");
+
+	($method_cfg,$extra_disk) = setup_netinst($tmpdir, $arch);
+
+	$ps_url = preseed_create_guest($gho, '', CDROM=>1);
+    }
     else
     {
 	die "$method";
-- 
2.1.4