[PATCH 0/4] Smart/RPM related fixes

[PATCH 0/4] Smart/RPM related fixes

[Paul Eggleton]

Note: there is one patch in here that affects BAD_RECOMMENDATIONS
behaviour for ipk.


The following changes since commit ac5d20f4adac69ea1702694fb50849c9e465b443:

  systemd: Remove systemd_unitdir if systemd is not in distro features (2013-07-23 07:49:54 -0700)

are available in the git repository at:

  git://git.openembedded.org/openembedded-core-contrib paule/smart
  http://cgit.openembedded.org/cgit.cgi/openembedded-core-contrib/log/?h=paule/smart

Paul Eggleton (4):
  python-smartpm: fix sometimes reporting no provider instead of conflict
  python-smartpm: add ignore-recommends package flag
  classes/image: ensure BAD_RECOMMENDATIONS supports pre-renamed package names
  classes/rootfs_rpm: implement BAD_RECOMMENDATIONS for RPM

 meta/classes/image.bbclass                         |   1 +
 meta/classes/package_rpm.bbclass                   |  11 +-
 meta/classes/rootfs_ipk.bbclass                    |   2 -
 meta/classes/rootfs_rpm.bbclass                    |   3 +-
 meta/conf/bitbake.conf                             |   2 +
 .../python-smartpm/smart-conflict-provider.patch   | 196 +++++++++++++++++++++
 .../smart-flag-ignore-recommends.patch             |  60 +++++++
 .../python/python-smartpm_1.4.1.bb                 |   2 +
 8 files changed, 271 insertions(+), 6 deletions(-)
 create mode 100644 meta/recipes-devtools/python/python-smartpm/smart-conflict-provider.patch
 create mode 100644 meta/recipes-devtools/python/python-smartpm/smart-flag-ignore-recommends.patch

-- 
1.8.1.2

[PATCH 1/4] python-smartpm: fix sometimes reporting no provider instead of conflict

[Paul Eggleton]

If a requirement of a package is conflicted, depending on how the
solution is reached, Smart's transaction code may eliminate all
providers of the requirement and then error out because nothing provides
them without saying why. To work around this, store a reason and report
that back if we need to, so for example instead of:

  error: Can't install packagegroup-core-ssh-dropbear-1.0-r1@all: no package provides dropbear

we will now get:

  error: Can't install packagegroup-core-ssh-dropbear-1.0-r1@all: unable to install provider for dropbear:
  error:     dropbear-2013.58-r1.0@armv5te is conflicted by openssh-sshd-6.2p2-r0@armv5te

Fixes [YOCTO #4305].

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 .../python-smartpm/smart-conflict-provider.patch   | 196 +++++++++++++++++++++
 .../python/python-smartpm_1.4.1.bb                 |   1 +
 2 files changed, 197 insertions(+)
 create mode 100644 meta/recipes-devtools/python/python-smartpm/smart-conflict-provider.patch

diff --git a/meta/recipes-devtools/python/python-smartpm/smart-conflict-provider.patch b/meta/recipes-devtools/python/python-smartpm/smart-conflict-provider.patch
new file mode 100644
index 0000000..10a7447
--- /dev/null
+++ b/meta/recipes-devtools/python/python-smartpm/smart-conflict-provider.patch
@@ -0,0 +1,196 @@
+Report a reason when a dependency could not be installed because it is locked
+
+If a requirement of a package is conflicted, depending on how the
+solution is reached, the transaction code may eliminate all providers
+of the requirement and then error out because nothing provides them. To
+work around this, store a reason in the locked dict and report that back
+if we need to, so for example instead of:
+
+  error: Can't install packagegroup-core-ssh-dropbear-1.0-r1@all: no package provides dropbear
+
+we now get:
+
+  error: Can't install packagegroup-core-ssh-dropbear-1.0-r1@all: unable to install provider for dropbear:
+  error:     dropbear-2013.58-r1.0@armv5te is conflicted by openssh-sshd-6.2p2-r0@armv5te
+
+Upstream-Status: Pending
+
+Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
+---
+ smart/const.py       |  7 +++++++
+ smart/transaction.py | 58 +++++++++++++++++++++++++++++++++++++++++-----------
+ 2 files changed, 53 insertions(+), 12 deletions(-)
+
+diff --git a/smart/const.py b/smart/const.py
+index 4d8e5cb..67c1ac5 100644
+--- a/smart/const.py
++++ b/smart/const.py
+@@ -70,4 +70,11 @@ DATADIR     = "/var/lib/smart/"
+ USERDATADIR = "~/.smart/"
+ CONFFILE    = "config"
+ 
++LOCKED_INSTALL     = Enum('LOCKED_INSTALL')
++LOCKED_REMOVE      = Enum('LOCKED_REMOVE')
++LOCKED_CONFLICT    = Enum('LOCKED_CONFLICT')
++LOCKED_CONFLICT_BY = Enum('LOCKED_CONFLICT_BY')
++LOCKED_NO_COEXIST  = Enum('LOCKED_NO_COEXIST')
++LOCKED_SYSCONF     = Enum('LOCKED_SYSCONF')
++
+ # vim:ts=4:sw=4:et
+diff --git a/smart/transaction.py b/smart/transaction.py
+index 300b9cc..dd9aa38 100644
+--- a/smart/transaction.py
++++ b/smart/transaction.py
+@@ -19,10 +19,31 @@
+ # along with Smart Package Manager; if not, write to the Free Software
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ #
+-from smart.const import INSTALL, REMOVE, UPGRADE, FIX, REINSTALL, KEEP
++from smart.const import INSTALL, REMOVE, UPGRADE, FIX, REINSTALL, KEEP, LOCKED_INSTALL, LOCKED_CONFLICT, LOCKED_CONFLICT_BY, LOCKED_NO_COEXIST, LOCKED_SYSCONF, LOCKED_REMOVE
+ from smart.cache import PreRequires, Package
+ from smart import *
+ 
++def lock_reason(pkg, lockvalue):
++    try:
++        (reason, otherpkg) = lockvalue
++    except TypeError:
++        reason = None
++        lockvalue = None
++    if reason == LOCKED_INSTALL:
++        return _("%s is to be installed") % pkg
++    elif reason == LOCKED_CONFLICT:
++        return _("%s conflicts with %s") % (pkg, otherpkg)
++    elif reason ==  LOCKED_CONFLICT_BY:
++        return _("%s is conflicted by %s") % (pkg, otherpkg)
++    elif reason ==  LOCKED_NO_COEXIST:
++        return _("%s cannot coexist with %s") % (pkg, otherpkg)
++    elif reason ==  LOCKED_SYSCONF:
++        return _("%s is locked in system configuration") % pkg
++    elif reason ==  LOCKED_REMOVE:
++        return _("%s is to be removed") % pkg
++    else:
++        return _("%s is locked (unknown reason)") % pkg
++
+ class ChangeSet(dict):
+ 
+     def __init__(self, cache, state=None, requested=None):
+@@ -187,7 +208,7 @@ class Policy(object):
+         for pkg in pkgconf.filterByFlag("lock", cache.getPackages()):
+             if pkg not in self._locked:
+                 self._sysconflocked.append(pkg)
+-                self._locked[pkg] = True
++                self._locked[pkg] = (LOCKED_SYSCONF, None)
+ 
+     def runFinished(self):
+         self._priorities.clear()
+@@ -524,7 +545,7 @@ class Transaction(object):
+         if ownpending:
+             pending = []
+ 
+-        locked[pkg] = True
++        locked[pkg] = (LOCKED_INSTALL, None)
+         changeset.set(pkg, INSTALL)
+         isinst = changeset.installed
+ 
+@@ -535,7 +556,7 @@ class Transaction(object):
+                     if prvpkg is pkg:
+                         continue
+                     if not isinst(prvpkg):
+-                        locked[prvpkg] = True
++                        locked[prvpkg] = (LOCKED_CONFLICT_BY, pkg)
+                         continue
+                     if prvpkg in locked:
+                         raise Failed, _("Can't install %s: conflicted package "
+@@ -550,7 +571,7 @@ class Transaction(object):
+                     if cnfpkg is pkg:
+                         continue
+                     if not isinst(cnfpkg):
+-                        locked[cnfpkg] = True
++                        locked[cnfpkg] = (LOCKED_CONFLICT, pkg)
+                         continue
+                     if cnfpkg in locked:
+                         raise Failed, _("Can't install %s: it's conflicted by "
+@@ -565,7 +586,7 @@ class Transaction(object):
+         for namepkg in namepkgs:
+             if namepkg is not pkg and not pkg.coexists(namepkg):
+                 if not isinst(namepkg):
+-                    locked[namepkg] = True
++                    locked[namepkg] = (LOCKED_NO_COEXIST, pkg)
+                     continue
+                 if namepkg in locked:
+                     raise Failed, _("Can't install %s: it can't coexist "
+@@ -577,6 +598,7 @@ class Transaction(object):
+ 
+             # Check if someone is already providing it.
+             prvpkgs = {}
++            lockedpkgs = {}
+             found = False
+             for prv in req.providedby:
+                 for prvpkg in prv.packages:
+@@ -585,6 +607,8 @@ class Transaction(object):
+                         break
+                     if prvpkg not in locked:
+                         prvpkgs[prvpkg] = True
++                    else:
++                        lockedpkgs[prvpkg] = locked[prvpkg]
+                 else:
+                     continue
+                 break
+@@ -597,7 +621,17 @@ class Transaction(object):
+             if not prvpkgs:
+                 # No packages provide it at all. Give up.
+                 if req in pkg.requires:
+-                    raise Failed, _("Can't install %s: no package provides %s") % \
++                    reasons = []
++                    for prv in req.providedby:
++                        for prvpkg in prv.packages:
++                            lockedres = lockedpkgs.get(prvpkg, None)
++                            if lockedres:
++                                reasons.append(lock_reason(prvpkg, lockedres))
++                    if reasons:
++                        raise Failed, _("Can't install %s: unable to install provider for %s:\n    %s") % \
++                                (pkg, req, '\n    '.join(reasons))
++                    else:
++                        raise Failed, _("Can't install %s: no package provides %s") % \
+                                 (pkg, req)
+                 else:
+                     # It's only a recommend, skip
+@@ -627,7 +661,7 @@ class Transaction(object):
+         if ownpending:
+             pending = []
+ 
+-        locked[pkg] = True
++        locked[pkg] = (LOCKED_REMOVE, None)
+         changeset.set(pkg, REMOVE)
+         isinst = changeset.installed
+ 
+@@ -1140,22 +1174,22 @@ class Transaction(object):
+                 if op is KEEP:
+                     if pkg in changeset:
+                         del changeset[pkg]
+-                    locked[pkg] = True
++                    locked[pkg] = (LOCKED_KEEP, None)
+                 elif op is INSTALL:
+                     if not isinst(pkg) and pkg in locked:
+                         raise Failed, _("Can't install %s: it's locked") % pkg
+                     changeset.set(pkg, INSTALL)
+-                    locked[pkg] = True
++                    locked[pkg] = (LOCKED_INSTALL, None)
+                 elif op is REMOVE:
+                     if isinst(pkg) and pkg in locked:
+                         raise Failed, _("Can't remove %s: it's locked") % pkg
+                     changeset.set(pkg, REMOVE)
+-                    locked[pkg] = True
++                    locked[pkg] = (LOCKED_REMOVE, None)
+                 elif op is REINSTALL:
+                     if pkg in locked:
+                         raise Failed, _("Can't reinstall %s: it's locked")%pkg
+                     changeset.set(pkg, INSTALL, force=True)
+-                    locked[pkg] = True
++                    locked[pkg] = (LOCKED_INSTALL, None)
+                 elif op is UPGRADE:
+                     pass
+ 
+-- 
+1.8.1.2
+
diff --git a/meta/recipes-devtools/python/python-smartpm_1.4.1.bb b/meta/recipes-devtools/python/python-smartpm_1.4.1.bb
index 0cf3847..3e39231 100644
--- a/meta/recipes-devtools/python/python-smartpm_1.4.1.bb
+++ b/meta/recipes-devtools/python/python-smartpm_1.4.1.bb
@@ -26,6 +26,7 @@ SRC_URI = "\
           file://smart-multilib-fixes.patch \
           file://smart-yaml-error.patch \
           file://smart-channelsdir.patch \
+          file://smart-conflict-provider.patch \
           "
 
 SRC_URI[md5sum] = "573ef32ba177a6b3c4bf7ef04873fcb6"
-- 
1.8.1.2

[PATCH 2/4] python-smartpm: add ignore-recommends package flag

[Paul Eggleton]

Adds support for a flag that is saved into Smart's configuration against
a package that says it should not be installed if it is only recommended
by a package being installed rather than required. This will enable us
to add BAD_RECOMMENDATIONS support for RPM.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 .../smart-flag-ignore-recommends.patch             | 60 ++++++++++++++++++++++
 .../python/python-smartpm_1.4.1.bb                 |  1 +
 2 files changed, 61 insertions(+)
 create mode 100644 meta/recipes-devtools/python/python-smartpm/smart-flag-ignore-recommends.patch

diff --git a/meta/recipes-devtools/python/python-smartpm/smart-flag-ignore-recommends.patch b/meta/recipes-devtools/python/python-smartpm/smart-flag-ignore-recommends.patch
new file mode 100644
index 0000000..5d5c6f4
--- /dev/null
+++ b/meta/recipes-devtools/python/python-smartpm/smart-flag-ignore-recommends.patch
@@ -0,0 +1,60 @@
+Add ignore-recommends flag support
+
+Allow configuring recommends on specific packages to be ignored.
+
+Upstream-Status: Pending
+
+Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
+---
+ smart/commands/flag.py | 3 +++
+ smart/transaction.py   | 7 ++++++-
+ 2 files changed, 9 insertions(+), 1 deletion(-)
+
+diff --git a/smart/commands/flag.py b/smart/commands/flag.py
+index 8b90496..191bb11 100644
+--- a/smart/commands/flag.py
++++ b/smart/commands/flag.py
+@@ -47,6 +47,9 @@ Currently known flags are:
+   multi-version - Flagged packages may have more than one version
+                   installed in the system at the same time
+                   (backend dependent).
++  ignore-recommends - Flagged packages will not be installed, if
++                  they are only recommended by a package to be
++                  installed rather than required.
+ 
+   security      - Flagged packages are updates for security errata.
+   bugfix        - Flagged packages are updates for bugfix errata.
+diff --git a/smart/transaction.py b/smart/transaction.py
+index dd9aa38..38eabae 100644
+--- a/smart/transaction.py
++++ b/smart/transaction.py
+@@ -596,12 +596,17 @@ class Transaction(object):
+         # Install packages required by this one.
+         for req in pkg.requires + pkg.recommends:
+ 
++            reqrequired = req in pkg.requires
++
+             # Check if someone is already providing it.
+             prvpkgs = {}
+             lockedpkgs = {}
+             found = False
+             for prv in req.providedby:
+                 for prvpkg in prv.packages:
++                    if not reqrequired:
++                        if pkgconf.testFlag("ignore-recommends", prvpkg):
++                            continue
+                     if isinst(prvpkg):
+                         found = True
+                         break
+@@ -620,7 +625,7 @@ class Transaction(object):
+ 
+             if not prvpkgs:
+                 # No packages provide it at all. Give up.
+-                if req in pkg.requires:
++                if reqrequired:
+                     reasons = []
+                     for prv in req.providedby:
+                         for prvpkg in prv.packages:
+-- 
+1.8.1.2
+
diff --git a/meta/recipes-devtools/python/python-smartpm_1.4.1.bb b/meta/recipes-devtools/python/python-smartpm_1.4.1.bb
index 3e39231..70ac8bf 100644
--- a/meta/recipes-devtools/python/python-smartpm_1.4.1.bb
+++ b/meta/recipes-devtools/python/python-smartpm_1.4.1.bb
@@ -27,6 +27,7 @@ SRC_URI = "\
           file://smart-yaml-error.patch \
           file://smart-channelsdir.patch \
           file://smart-conflict-provider.patch \
+          file://smart-flag-ignore-recommends.patch \
           "
 
 SRC_URI[md5sum] = "573ef32ba177a6b3c4bf7ef04873fcb6"
-- 
1.8.1.2

[PATCH 3/4] classes/image: ensure BAD_RECOMMENDATIONS supports pre-renamed package names

[Paul Eggleton]

Use runtime remapping on BAD_RECOMMENDATIONS just as we do with
variables such as IMAGE_INSTALL, so that we're specifying the name prior
to any renaming e.g. that done by debian.bbclass. Note that this is a
change in behaviour for renamed packages, however this is the correct
thing to be doing; a search of existing layers suggests this shouldn't
cause widespread incompatibilities.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 meta/classes/image.bbclass | 1 +
 1 file changed, 1 insertion(+)

diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index 380ed8e..cf02f88 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -132,6 +132,7 @@ python () {
         pn = d.getVar('PN', True)
         runtime_mapping_rename("PACKAGE_INSTALL", pn, d)
         runtime_mapping_rename("PACKAGE_INSTALL_ATTEMPTONLY", pn, d)
+        runtime_mapping_rename("BAD_RECOMMENDATIONS", pn, d)
 
     # Ensure we have the vendor list for complementary package handling
     ml_vendor_list = ""
-- 
1.8.1.2

[PATCH 4/4] classes/rootfs_rpm: implement BAD_RECOMMENDATIONS for RPM

[Paul Eggleton]

Add support for the BAD_RECOMMENDATIONS variable that can be used to
prevent specific packages from being installed via an RRECOMMENDS
relationship when using the RPM backend. (Previously this
functionality was only available when using ipk packaging.)

In the process this moves the defaulting of BAD_RECOMMENDATIONS (as
empty) to bitbake.conf since it is no longer specific to the ipk
backend, as well as unifying some of the code that creates the
configuration for smart for use on the host and target.

Fixes [YOCTO #3916].

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 meta/classes/package_rpm.bbclass | 11 +++++++++--
 meta/classes/rootfs_ipk.bbclass  |  2 --
 meta/classes/rootfs_rpm.bbclass  |  3 +--
 meta/conf/bitbake.conf           |  2 ++
 4 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/meta/classes/package_rpm.bbclass b/meta/classes/package_rpm.bbclass
index 1537079..324d83f 100644
--- a/meta/classes/package_rpm.bbclass
+++ b/meta/classes/package_rpm.bbclass
@@ -224,6 +224,14 @@ translate_oe_to_smart() {
 	export pkgs_to_install
 }
 
+package_write_smart_config() {
+	# Write common configuration for host and target usage
+	smart --data-dir=$1/var/lib/smart config --set rpm-nolinktos=1
+	smart --data-dir=$1/var/lib/smart config --set rpm-noparentdirs=1
+	for i in ${BAD_RECOMMENDATIONS}; do
+		smart --data-dir=$1/var/lib/smart flag --set ignore-recommends $i
+	done
+}
 
 #
 # Install a bunch of packages using rpm.
@@ -345,10 +353,9 @@ EOF
 		rm -rf ${target_rootfs}/var/lib/smart
 		smart --data-dir=${target_rootfs}/var/lib/smart config --set rpm-root=${target_rootfs}
 		smart --data-dir=${target_rootfs}/var/lib/smart config --set rpm-dbpath=${rpmlibdir}
-		smart --data-dir=${target_rootfs}/var/lib/smart config --set rpm-nolinktos=1
-		smart --data-dir=${target_rootfs}/var/lib/smart config --set rpm-noparentdirs=1
 		smart --data-dir=${target_rootfs}/var/lib/smart config --set rpm-extra-macros._var=${localstatedir}
 		smart --data-dir=${target_rootfs}/var/lib/smart config --set rpm-extra-macros._tmppath=/install/tmp
+		package_write_smart_config ${target_rootfs}
 		# Optional debugging
 		#smart --data-dir=${target_rootfs}/var/lib/smart config --set rpm-log-level=debug
 		#smart --data-dir=${target_rootfs}/var/lib/smart config --set rpm-log-file=/tmp/smart-debug-logfile
diff --git a/meta/classes/rootfs_ipk.bbclass b/meta/classes/rootfs_ipk.bbclass
index 1403422..c9650cb 100644
--- a/meta/classes/rootfs_ipk.bbclass
+++ b/meta/classes/rootfs_ipk.bbclass
@@ -21,8 +21,6 @@ OPKG_POSTPROCESS_COMMANDS = "ipk_insert_feed_uris; "
 
 OPKGLIBDIR = "${localstatedir}/lib"
 
-# Which packages to not install on the basis of a recommendation
-BAD_RECOMMENDATIONS ?= ""
 MULTILIBRE_ALLOW_REP = "${OPKGLIBDIR}/opkg"
 
 fakeroot rootfs_ipk_do_rootfs () {
diff --git a/meta/classes/rootfs_rpm.bbclass b/meta/classes/rootfs_rpm.bbclass
index 1b09c8d..30b52d4 100644
--- a/meta/classes/rootfs_rpm.bbclass
+++ b/meta/classes/rootfs_rpm.bbclass
@@ -163,8 +163,7 @@ rpm_setup_smart_target_config() {
 	# Set up smart configuration for the target
 	rm -rf ${IMAGE_ROOTFS}/var/lib/smart
 	smart --data-dir=${IMAGE_ROOTFS}/var/lib/smart channel --add rpmsys type=rpm-sys -y
-	smart --data-dir=${IMAGE_ROOTFS}/var/lib/smart config --set rpm-nolinktos=1
-	smart --data-dir=${IMAGE_ROOTFS}/var/lib/smart config --set rpm-noparentdirs=1
+	package_write_smart_config ${IMAGE_ROOTFS}
 	rm -f ${IMAGE_ROOTFS}/var/lib/smart/config.old
 }
 
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index 62a3936..07eb473 100644
--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -753,6 +753,8 @@ COMBINED_FEATURES[vardeps] += "DISTRO_FEATURES MACHINE_FEATURES"
 SERIAL_CONSOLE ??= ""
 SERIAL_CONSOLES ??= "${@d.getVar('SERIAL_CONSOLE', True).replace(' ', ';')}"
 
+BAD_RECOMMENDATIONS ?= ""
+
 # Make sure MACHINE isn't exported
 # (breaks binutils at least)
 MACHINE[unexport] = "1"
-- 
1.8.1.2