[PATCH 01/10] wic: Remove dependency on rt_util module

[PATCH 01/10] wic: Remove dependency on rt_util module

[Tom Zanussi]

rt_util contains bootstrap_mic(), which imports rpm and other things
we don't need because we don't do bootstrap i.e. runtime (set in
wic.conf) is always set to 'native', which means use what's on the
local host.

bootstrap mode is for downloading and installing rpms that wic needs,
which we may want to implement later; for now, we just want to use
what's local.

Signed-off-by: Tom Zanussi <tom.zanussi@linux.intel.com>
---
 scripts/lib/mic/creator.py                       | 2 +-
 scripts/lib/mic/plugins/imager/direct_plugin.py  | 2 +-
 scripts/lib/mic/plugins/imager/fs_plugin.py      | 2 +-
 scripts/lib/mic/plugins/imager/livecd_plugin.py  | 2 +-
 scripts/lib/mic/plugins/imager/liveusb_plugin.py | 2 +-
 scripts/lib/mic/plugins/imager/loop_plugin.py    | 2 +-
 scripts/lib/mic/plugins/imager/raw_plugin.py     | 2 +-
 7 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/scripts/lib/mic/creator.py b/scripts/lib/mic/creator.py
index f3d0de1..b36bbf0 100644
--- a/scripts/lib/mic/creator.py
+++ b/scripts/lib/mic/creator.py
@@ -18,7 +18,7 @@
 import os, sys, re
 from optparse import SUPPRESS_HELP
 
-from mic import msger, rt_util
+from mic import msger
 from mic.utils import cmdln, errors, rpmmisc
 from mic.conf import configmgr
 from mic.plugin import pluginmgr
diff --git a/scripts/lib/mic/plugins/imager/direct_plugin.py b/scripts/lib/mic/plugins/imager/direct_plugin.py
index 53381e5..e015256 100644
--- a/scripts/lib/mic/plugins/imager/direct_plugin.py
+++ b/scripts/lib/mic/plugins/imager/direct_plugin.py
@@ -30,7 +30,7 @@ import shutil
 import re
 import tempfile
 
-from mic import chroot, msger, rt_util
+from mic import chroot, msger
 from mic.utils import misc, fs_related, errors, runner, cmdln
 from mic.conf import configmgr
 from mic.plugin import pluginmgr
diff --git a/scripts/lib/mic/plugins/imager/fs_plugin.py b/scripts/lib/mic/plugins/imager/fs_plugin.py
index 8e758db..6bcaf00 100644
--- a/scripts/lib/mic/plugins/imager/fs_plugin.py
+++ b/scripts/lib/mic/plugins/imager/fs_plugin.py
@@ -18,7 +18,7 @@
 import os
 import sys
 
-from mic import chroot, msger, rt_util
+from mic import chroot, msger
 from mic.utils import cmdln, misc, errors, fs_related
 from mic.imager import fs
 from mic.conf import configmgr
diff --git a/scripts/lib/mic/plugins/imager/livecd_plugin.py b/scripts/lib/mic/plugins/imager/livecd_plugin.py
index d24ef59..82cb1af 100644
--- a/scripts/lib/mic/plugins/imager/livecd_plugin.py
+++ b/scripts/lib/mic/plugins/imager/livecd_plugin.py
@@ -19,7 +19,7 @@ import os
 import shutil
 import tempfile
 
-from mic import chroot, msger, rt_util
+from mic import chroot, msger
 from mic.utils import misc, fs_related, errors
 from mic.conf import configmgr
 import mic.imager.livecd as livecd
diff --git a/scripts/lib/mic/plugins/imager/liveusb_plugin.py b/scripts/lib/mic/plugins/imager/liveusb_plugin.py
index 7aa8927..3d53c84 100644
--- a/scripts/lib/mic/plugins/imager/liveusb_plugin.py
+++ b/scripts/lib/mic/plugins/imager/liveusb_plugin.py
@@ -19,7 +19,7 @@ import os
 import shutil
 import tempfile
 
-from mic import chroot, msger, rt_util
+from mic import chroot, msger
 from mic.utils import misc, fs_related, errors
 from mic.utils.partitionedfs import PartitionedMount
 from mic.conf import configmgr
diff --git a/scripts/lib/mic/plugins/imager/loop_plugin.py b/scripts/lib/mic/plugins/imager/loop_plugin.py
index 8f4b030..2a05b3c 100644
--- a/scripts/lib/mic/plugins/imager/loop_plugin.py
+++ b/scripts/lib/mic/plugins/imager/loop_plugin.py
@@ -19,7 +19,7 @@ import os
 import shutil
 import tempfile
 
-from mic import chroot, msger, rt_util
+from mic import chroot, msger
 from mic.utils import misc, fs_related, errors, cmdln
 from mic.conf import configmgr
 from mic.plugin import pluginmgr
diff --git a/scripts/lib/mic/plugins/imager/raw_plugin.py b/scripts/lib/mic/plugins/imager/raw_plugin.py
index 1b9631d..f9625b8 100644
--- a/scripts/lib/mic/plugins/imager/raw_plugin.py
+++ b/scripts/lib/mic/plugins/imager/raw_plugin.py
@@ -20,7 +20,7 @@ import shutil
 import re
 import tempfile
 
-from mic import chroot, msger, rt_util
+from mic import chroot, msger
 from mic.utils import misc, fs_related, errors, runner, cmdln
 from mic.conf import configmgr
 from mic.plugin import pluginmgr
-- 
1.8.3.1

[PATCH 02/10] wic: eliminate module checks

[Tom Zanussi]

We're removing all external dependencies including rpm and urlgrabber,
so we don't need this check.

Signed-off-by: Tom Zanussi <tom.zanussi@linux.intel.com>
---
 scripts/wic | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/scripts/wic b/scripts/wic
index 08473d3..b6fd16c 100755
--- a/scripts/wic
+++ b/scripts/wic
@@ -38,16 +38,6 @@ import optparse
 import logging
 
 # External modules
-try:
-    import rpm
-    import urlgrabber
-except ImportError:
-    print("ERROR: failed to import necessary modules.")
-    print("Please ensure the following modules are installed:")
-    print("\trpm");
-    print("\turlgrabber");
-    sys.exit(1)
-
 scripts_path = os.path.abspath(os.path.dirname(os.path.abspath(sys.argv[0])))
 lib_path = scripts_path + '/lib'
 sys.path = sys.path + [lib_path]
-- 
1.8.3.1

[PATCH 03/10] wic: Remove dependency on rpmmisc

[Tom Zanussi]

rpmmisc imports rpm and contains misc rpm utilities related to
packaging and determining arches based on the packaging.  We should
never run across this in the initial version of wic, so remove the
dependency.

Signed-off-by: Tom Zanussi <tom.zanussi@linux.intel.com>
---
 scripts/lib/mic/creator.py       | 2 +-
 scripts/lib/mic/imager/livecd.py | 2 +-
 scripts/lib/mic/utils/misc.py    | 1 -
 3 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/scripts/lib/mic/creator.py b/scripts/lib/mic/creator.py
index b36bbf0..267928f 100644
--- a/scripts/lib/mic/creator.py
+++ b/scripts/lib/mic/creator.py
@@ -19,7 +19,7 @@ import os, sys, re
 from optparse import SUPPRESS_HELP
 
 from mic import msger
-from mic.utils import cmdln, errors, rpmmisc
+from mic.utils import cmdln, errors
 from mic.conf import configmgr
 from mic.plugin import pluginmgr
 
diff --git a/scripts/lib/mic/imager/livecd.py b/scripts/lib/mic/imager/livecd.py
index a992ee0..0711552 100644
--- a/scripts/lib/mic/imager/livecd.py
+++ b/scripts/lib/mic/imager/livecd.py
@@ -20,7 +20,7 @@ import glob
 import shutil
 
 from mic import kickstart, msger
-from mic.utils import fs_related, rpmmisc, runner, misc
+from mic.utils import fs_related, runner, misc
 from mic.utils.errors import CreatorError
 from mic.imager.loop import LoopImageCreator
 
diff --git a/scripts/lib/mic/utils/misc.py b/scripts/lib/mic/utils/misc.py
index 67ddef2..2c4a3ac 100644
--- a/scripts/lib/mic/utils/misc.py
+++ b/scripts/lib/mic/utils/misc.py
@@ -45,7 +45,6 @@ from mic.utils.fs_related import find_binary_path, makedirs
 from mic.utils.grabber import myurlgrab
 from mic.utils.proxy import get_proxy_for
 from mic.utils import runner
-from mic.utils import rpmmisc
 
 
 RPM_RE  = re.compile("(.*)\.(.*) (.*)-(.*)")
-- 
1.8.3.1

[PATCH 00/10] wic dependency and rootfs updates

[Tom Zanussi]

This patchset mainly removes unnecessary run-time python module and
executable dependencies for mic functionality we currently don't use.
I didn't originally hit these myself because I did development on
Fedora and Ubuntu users hit these - I've tested the resulting patchset
on both Fedora and Ubuntu with the affected packages removed and don't
see any problems.

I've kept them as a series of changes rather than glomming them all
together because at some point we will want to restore these
dependencies on a case-by-case basis and it's better to have the
changes as individually explained commits.

It also removes and unncessary pseudo export as discussed in a
previous thread, and moves the setup to the top-level rootfs
generation function.

The following changes since commit 505114c13d29799a25379f7ce720d632616d8776:

  wic: add pseudo to the populate-extfs step (2013-10-18 16:03:39 +0100)

are available in the git repository at:

  git://git.yoctoproject.org/poky-contrib.git tzanussi/wic-fixes-3
  http://git.yoctoproject.org/cgit/cgit.cgi/poky-contrib/log/?h=tzanussi/wic-fixes-3

Tom Zanussi (10):
  wic: Remove dependency on rt_util module
  wic: eliminate module checks
  wic: Remove dependency on rpmmisc
  wic: Remove dependency on myurlgrab module
  wic: remove rpm warning code from BackendPlugin
  wic: Remove rpm and grabber dependencies from BaseImageCreator
  wic: Remove rpmmisc call from livecd
  wic: Remove binary dependencies
  wic: Make find_binary_path() more user-friendly
  wic: Update and generalize pseudo setup for rootfs generation

 scripts/lib/mic/creator.py                         |  4 ++--
 scripts/lib/mic/imager/baseimager.py               |  4 +---
 scripts/lib/mic/imager/livecd.py                   |  4 ++--
 .../lib/mic/kickstart/custom_commands/partition.py | 28 ++++++++++++----------
 scripts/lib/mic/pluginbase.py                      |  5 ----
 scripts/lib/mic/plugins/imager/direct_plugin.py    |  2 +-
 scripts/lib/mic/plugins/imager/fs_plugin.py        |  2 +-
 scripts/lib/mic/plugins/imager/livecd_plugin.py    |  2 +-
 scripts/lib/mic/plugins/imager/liveusb_plugin.py   |  2 +-
 scripts/lib/mic/plugins/imager/loop_plugin.py      |  2 +-
 scripts/lib/mic/plugins/imager/raw_plugin.py       |  2 +-
 scripts/lib/mic/utils/fs_related.py                |  5 +++-
 scripts/lib/mic/utils/misc.py                      |  2 --
 scripts/lib/mic/utils/partitionedfs.py             |  4 ----
 scripts/wic                                        | 10 --------
 15 files changed, 30 insertions(+), 48 deletions(-)

-- 
1.8.3.1

[PATCH 04/10] wic: Remove dependency on myurlgrab module

[Tom Zanussi]

mylrlgrab is in grabber, which imports rpm.  For current
functionality, we don't need to grab urls or import rpm, so remove the
dependency.

Signed-off-by: Tom Zanussi <tom.zanussi@linux.intel.com>
---
 scripts/lib/mic/utils/misc.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/lib/mic/utils/misc.py b/scripts/lib/mic/utils/misc.py
index 2c4a3ac..95241d7 100644
--- a/scripts/lib/mic/utils/misc.py
+++ b/scripts/lib/mic/utils/misc.py
@@ -42,7 +42,6 @@ xmlparse = cElementTree.parse
 from mic import msger
 from mic.utils.errors import CreatorError, SquashfsError
 from mic.utils.fs_related import find_binary_path, makedirs
-from mic.utils.grabber import myurlgrab
 from mic.utils.proxy import get_proxy_for
 from mic.utils import runner
 
-- 
1.8.3.1

[PATCH 05/10] wic: remove rpm warning code from BackendPlugin

[Tom Zanussi]

We don't currently use rpm functionality, so we don't need to silence
rpm warnings.

Signed-off-by: Tom Zanussi <tom.zanussi@linux.intel.com>
---
 scripts/lib/mic/pluginbase.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/scripts/lib/mic/pluginbase.py b/scripts/lib/mic/pluginbase.py
index 6ac195b..2f9d720 100644
--- a/scripts/lib/mic/pluginbase.py
+++ b/scripts/lib/mic/pluginbase.py
@@ -83,11 +83,6 @@ class ImagerPlugin(_Plugin):
 class BackendPlugin(_Plugin):
     mic_plugin_type="backend"
 
-    # suppress the verbose rpm warnings
-    if msger.get_loglevel() != 'debug':
-        import rpm
-        rpm.setVerbosity(rpm.RPMLOG_ERR)
-
     def addRepository(self):
         pass
 
-- 
1.8.3.1

[PATCH 06/10] wic: Remove rpm and grabber dependencies from BaseImageCreator

[Tom Zanussi]

BaseImageCreator is a base class for DirectImageCreator and others,
and imports rpm and grabber (which imports rpm).

The various plugins e.g. DirectPlugin import the creators and
therefore these dependencies, which manifest at run-time as e.g.:

  Warning: Failed to load plugin imager/direct_plugin: No module named
    rpm

Signed-off-by: Tom Zanussi <tom.zanussi@linux.intel.com>
---
 scripts/lib/mic/imager/baseimager.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/scripts/lib/mic/imager/baseimager.py b/scripts/lib/mic/imager/baseimager.py
index 4d6be29..b721249 100644
--- a/scripts/lib/mic/imager/baseimager.py
+++ b/scripts/lib/mic/imager/baseimager.py
@@ -26,12 +26,10 @@ import re
 import tarfile
 import glob
 
-import rpm
-
 from mic import kickstart
 from mic import msger
 from mic.utils.errors import CreatorError, Abort
-from mic.utils import misc, grabber, runner, fs_related as fs
+from mic.utils import misc, runner, fs_related as fs
 
 class BaseImageCreator(object):
     """Installs a system to a chroot directory.
-- 
1.8.3.1

[PATCH 07/10] wic: Remove rpmmisc call from livecd

[Tom Zanussi]

We don't currently use LiveCDImageCreator, but it makes calls when
initialized via the plugin interface to rpmmisc module functions,
which we don't want the dependency on.

To make it (and LiveUSBImageCreator) happy, we give it the dummy
"i386" value for now.

Signed-off-by: Tom Zanussi <tom.zanussi@linux.intel.com>
---
 scripts/lib/mic/imager/livecd.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/lib/mic/imager/livecd.py b/scripts/lib/mic/imager/livecd.py
index 0711552..e36f4a7 100644
--- a/scripts/lib/mic/imager/livecd.py
+++ b/scripts/lib/mic/imager/livecd.py
@@ -741,7 +741,7 @@ hiddenmenu
         self._configure_syslinux_bootloader(isodir)
         self._configure_efi_bootloader(isodir)
 
-arch = rpmmisc.getBaseArch()
+arch = "i386"
 if arch in ("i386", "x86_64"):
     LiveCDImageCreator = x86LiveImageCreator
 elif arch.startswith("arm"):
-- 
1.8.3.1

[PATCH 08/10] wic: Remove binary dependencies

[Tom Zanussi]

Current functionality doesn't make use of kpartx, mount, or unmount,
and we use native mkswap, so remove the binary checks for those.

Signed-off-by: Tom Zanussi <tom.zanussi@linux.intel.com>
---
 scripts/lib/mic/utils/partitionedfs.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/scripts/lib/mic/utils/partitionedfs.py b/scripts/lib/mic/utils/partitionedfs.py
index 59263ba..6607466 100644
--- a/scripts/lib/mic/utils/partitionedfs.py
+++ b/scripts/lib/mic/utils/partitionedfs.py
@@ -45,11 +45,7 @@ class PartitionedMount(Mount):
         self.mountOrder = []
         self.unmountOrder = []
         self.parted = find_binary_path("parted")
-        self.kpartx = find_binary_path("kpartx")
-        self.mkswap = find_binary_path("mkswap")
         self.btrfscmd=None
-        self.mountcmd = find_binary_path("mount")
-        self.umountcmd = find_binary_path("umount")
         self.skipformat = skipformat
         self.snapshot_created = self.skipformat
         # Size of a sector used in calculations
-- 
1.8.3.1

[PATCH 09/10] wic: Make find_binary_path() more user-friendly

[Tom Zanussi]

find_binary_path() is useful, but if the binary isn't found, it prints
a stacktrace and a less-than-useful message.  Users complain when they
get stacktraces for things they can act on, so remove the stacktrace
and tell the user what the problem is.

Signed-off-by: Tom Zanussi <tom.zanussi@linux.intel.com>
---
 scripts/lib/mic/utils/fs_related.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scripts/lib/mic/utils/fs_related.py b/scripts/lib/mic/utils/fs_related.py
index 6161735..dd420e8 100644
--- a/scripts/lib/mic/utils/fs_related.py
+++ b/scripts/lib/mic/utils/fs_related.py
@@ -57,7 +57,10 @@ def find_binary_path(binary):
         bin_path = "%s/%s" % (path, binary)
         if os.path.exists(bin_path):
             return bin_path
-    raise CreatorError("Command '%s' is not available." % binary)
+
+    print "External command '%s' not found, exiting." % binary
+    print "  (Please install '%s' on your host system)" % binary
+    sys.exit(1)
 
 def makedirs(dirname):
     """A version of os.makedirs() that doesn't throw an
-- 
1.8.3.1

[PATCH 10/10] wic: Update and generalize pseudo setup for rootfs generation

[Tom Zanussi]

Remove unnecessary pseudo exports i.e. PSEUDO_DISABLED and move the
setup to the top-level prepare_rootfs().

Signed-off-by: Tom Zanussi <tom.zanussi@linux.intel.com>
---
 .../lib/mic/kickstart/custom_commands/partition.py | 28 ++++++++++++----------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/scripts/lib/mic/kickstart/custom_commands/partition.py b/scripts/lib/mic/kickstart/custom_commands/partition.py
index 0eb0671..4b11195 100644
--- a/scripts/lib/mic/kickstart/custom_commands/partition.py
+++ b/scripts/lib/mic/kickstart/custom_commands/partition.py
@@ -204,25 +204,27 @@ class Wic_PartData(Mic_PartData):
 
         Currently handles ext2/3/4 and btrfs.
         """
+        pseudo = "export PSEUDO_PREFIX=%s/usr;" % native_sysroot
+        pseudo += "export PSEUDO_LOCALSTATEDIR=%s/../pseudo;" % rootfs_dir
+        pseudo += "export PSEUDO_PASSWD=%s;" % rootfs_dir
+        pseudo += "export PSEUDO_NOSYMLINKEXP=1;"
+        pseudo += "%s/usr/bin/pseudo " % native_sysroot
+
         if self.fstype.startswith("ext"):
             return self.prepare_rootfs_ext(cr_workdir, oe_builddir,
-                                           rootfs_dir, native_sysroot)
+                                           rootfs_dir, native_sysroot,
+                                           pseudo)
         elif self.fstype.startswith("btrfs"):
             return self.prepare_rootfs_btrfs(cr_workdir, oe_builddir,
-                                             rootfs_dir, native_sysroot)
+                                             rootfs_dir, native_sysroot,
+                                             pseudo)
 
     def prepare_rootfs_ext(self, cr_workdir, oe_builddir, rootfs_dir,
-                           native_sysroot):
+                           native_sysroot, pseudo):
         """
         Prepare content for an ext2/3/4 rootfs partition.
         """
-        populate_script = "export PSEUDO_PREFIX=%s/usr;" % native_sysroot
-        populate_script += "export PSEUDO_LOCALSTATEDIR=%s/../pseudo;" % rootfs_dir
-        populate_script += "export PSEUDO_PASSWD=%s;" % rootfs_dir
-        populate_script += "export PSEUDO_NOSYMLINKEXP=1;"
-        populate_script += "export PSEUDO_DISABLED=0;"
-        populate_script += "%s/usr/bin/pseudo %s/usr/bin/populate-extfs.sh" % \
-            (native_sysroot, native_sysroot)
+        populate_script = "%s/usr/bin/populate-extfs.sh" % native_sysroot
 
         image_extra_space = 10240
 
@@ -245,7 +247,7 @@ class Wic_PartData(Mic_PartData):
         rc, out = exec_native_cmd(mkfs_cmd, native_sysroot)
 
         populate_cmd = populate_script + " " + image_rootfs + " " + rootfs
-        rc, out = exec_native_cmd(populate_cmd, native_sysroot)
+        rc, out = exec_native_cmd(pseudo + populate_cmd, native_sysroot)
 
         # get the rootfs size in the right units for kickstart (Mb)
         du_cmd = "du -Lbms %s" % rootfs
@@ -258,7 +260,7 @@ class Wic_PartData(Mic_PartData):
         return 0
 
     def prepare_rootfs_btrfs(self, cr_workdir, oe_builddir, rootfs_dir,
-                             native_sysroot):
+                             native_sysroot, pseudo):
         """
         Prepare content for a btrfs rootfs partition.
 
@@ -281,7 +283,7 @@ class Wic_PartData(Mic_PartData):
 
         mkfs_cmd = "mkfs.%s -b %d -r %s %s" % \
             (self.fstype, rootfs_size * 1024, image_rootfs, rootfs)
-        rc, out = exec_native_cmd(mkfs_cmd, native_sysroot)
+        rc, out = exec_native_cmd(pseudo + mkfs_cmd, native_sysroot)
 
         # get the rootfs size in the right units for kickstart (Mb)
         du_cmd = "du -Lbms %s" % rootfs
-- 
1.8.3.1