[MULTILIB_SUBTASK4 0/5] opkg support for multilib

[MULTILIB_SUBTASK4 0/5] opkg support for multilib

[Lianhao Lu]

This series of patches is the subtask 4 of https://wiki.yoctoproject.org/wiki/Multilib.

It is based on the git branch of "poky-contrib/rpurdie/ml".

It added the support to opkg backend to allow install of multilib variant packages, also fixed some other bugs.

The multilib variant packages to be installed should be set in the variable MULTILIB_IMAGE_INSTALL.

The following changes since commit b04ee632eb06650dde3e3ee8c4788a45cae0aa5e:
  Richard Purdie (1):
        multilib: First pass from RP

are available in the git repository at:

  git://git.pokylinux.org/poky-contrib llu/ml
  http://git.pokylinux.org/cgit.cgi/poky-contrib/log/?h=llu/ml

Lianhao Lu (5):
  libc-package.bbclass: Replace hard coded libdir.
  classes/multilib.bbclass: Change the SHLIBSDIR.
  multilib/package.bbclass: Set PACKAGE_ARCH for multilib.
  image.bbclass: Added variables for multilib support.
  package(rootfs)_ipk.bbclass: support multilib in opkg backend.

 meta/classes/image.bbclass        |   54 ++++++++++++++++++++++++++++++++++++-
 meta/classes/libc-package.bbclass |    7 +++--
 meta/classes/multilib.bbclass     |   11 ++-----
 meta/classes/package.bbclass      |   11 +++++++
 meta/classes/package_ipk.bbclass  |   51 ++++++++++++++++++++++++++++++++++
 meta/classes/rootfs_ipk.bbclass   |    3 ++
 6 files changed, 125 insertions(+), 12 deletions(-)

[MULTILIB_SUBTASK4 1/5] libc-package.bbclass: Replace hard coded libdir.

[Lianhao Lu]

Replace the hard coded libdir for locale generating to meet the multilib
requirement.

Signed-off-by: Lianhao Lu <lianhao.lu@intel.com>
---
 meta/classes/libc-package.bbclass |    7 ++++---
 1 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/meta/classes/libc-package.bbclass b/meta/classes/libc-package.bbclass
index 55e3d48..5db46ce 100644
--- a/meta/classes/libc-package.bbclass
+++ b/meta/classes/libc-package.bbclass
@@ -277,10 +277,11 @@ python package_do_split_gconvs () {
 
 	def output_locale_binary(name, pkgname, locale, encoding):
 		treedir = base_path_join(bb.data.getVar("WORKDIR", d, 1), "locale-tree")
-		ldlibdir = "%s/lib" % treedir
+		ldlibdir = base_path_join(treedir, bb.data.getVar("base_libdir", d, 1))
 		path = bb.data.getVar("PATH", d, 1)
 		i18npath = base_path_join(treedir, datadir, "i18n")
 		gconvpath = base_path_join(treedir, "iconvdata")
+		outputpath = base_path_join(treedir, libdir, "locale")
 
 		use_cross_localedef = bb.data.getVar("LOCALE_GENERATION_WITH_CROSS-LOCALEDEF", d, 1) or "0"
 		if use_cross_localedef == "1":
@@ -300,8 +301,8 @@ python package_do_split_gconvs () {
 				raise bb.build.FuncFailed("unknown arch:" + target_arch + " for locale_arch_options")
 
 			localedef_opts += " --force --old-style --no-archive --prefix=%s \
-				--inputfile=%s/%s/i18n/locales/%s --charmap=%s %s/usr/lib/locale/%s" \
-				% (treedir, treedir, datadir, locale, encoding, treedir, name)
+				--inputfile=%s/%s/i18n/locales/%s --charmap=%s %s/%s" \
+				% (treedir, treedir, datadir, locale, encoding, outputpath, name)
 
 			cmd = "PATH=\"%s\" I18NPATH=\"%s\" GCONV_PATH=\"%s\" cross-localedef %s" % \
 				(path, i18npath, gconvpath, localedef_opts)
-- 
1.7.0.4

[MULTILIB_SUBTASK4 2/5] classes/multilib.bbclass: Change the SHLIBSDIR.

[Lianhao Lu]

Changes the SHLIBSDIR for the multilib version, so it won't mix up with
the normal version.

Signed-off-by: Lianhao Lu <lianhao.lu@intel.com>
---
 meta/classes/multilib.bbclass |    1 +
 1 files changed, 1 insertions(+), 0 deletions(-)

diff --git a/meta/classes/multilib.bbclass b/meta/classes/multilib.bbclass
index 36c52ef..17edc37 100644
--- a/meta/classes/multilib.bbclass
+++ b/meta/classes/multilib.bbclass
@@ -30,6 +30,7 @@ python multilib_virtclass_handler () {
             e.data.setVar("BASE_PACKAGE_ARCH", "${TARGET_ARCH}")
 
     bb.data.setVar("PN", variant + "-" + bb.data.getVar("PN", e.data, False), e.data)
+    bb.data.setVar("SHLIBSDIR_virtclass-multilib-" + variant ,bb.data.getVar("SHLIBSDIR",e.data,False) + "/" + variant, e.data)
     bb.data.setVar("OVERRIDES", bb.data.getVar("OVERRIDES", e.data, False) + override, e.data)
 }
 
-- 
1.7.0.4

[MULTILIB_SUBTASK4 3/5] multilib/package.bbclass: Set PACKAGE_ARCH for multilib.

[Lianhao Lu]

Set PACKAGE_ARCH in case of multilib.

Signed-off-by: Lianhao Lu <lianhao.lu@intel.com>
---
 meta/classes/multilib.bbclass |   10 ++--------
 meta/classes/package.bbclass  |   11 +++++++++++
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/meta/classes/multilib.bbclass b/meta/classes/multilib.bbclass
index 17edc37..66d2df9 100644
--- a/meta/classes/multilib.bbclass
+++ b/meta/classes/multilib.bbclass
@@ -21,16 +21,10 @@ python multilib_virtclass_handler () {
         e.data.delVarFlags("libdir_virtclass-multilib-" + variant)
         e.data.setVar("target_libdir", libdir)
 
-        tarch = e.data.getVar("TARGET_ARCH", True)
-        parch = e.data.getVar("PACKAGE_ARCH", True)
-        baseparch = e.data.getVar("BASE_PACKAGE_ARCH", True)
-        if parch == tarch:
-            e.data.setVar("PACKAGE_ARCH", "${TARGET_ARCH}")
-        if baseparch == tarch:
-            e.data.setVar("BASE_PACKAGE_ARCH", "${TARGET_ARCH}")
-
     bb.data.setVar("PN", variant + "-" + bb.data.getVar("PN", e.data, False), e.data)
     bb.data.setVar("SHLIBSDIR_virtclass-multilib-" + variant ,bb.data.getVar("SHLIBSDIR",e.data,False) + "/" + variant, e.data)
+    bb.data.setVar("PACKAGE_ARCH", "${BASE_PACKAGE_ARCH}" , e.data);
+    bb.data.setVar("BASE_PACKAGE_ARCH_virtclass-multilib-" + variant, "ml" + variant, e.data);
     bb.data.setVar("OVERRIDES", bb.data.getVar("OVERRIDES", e.data, False) + override, e.data)
 }
 
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index 8f91c95..2ad2cf2 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -166,6 +166,17 @@ python () {
         bb.data.setVarFlag('do_package', 'deptask', " ".join(deps), d)
     else:
         d.setVar("PACKAGERDEPTASK", "")
+
+    multilib_archs = []
+    multilibs= d.getVar('MULTILIBS', True) or ""
+    if multilibs:
+        for ext in multilibs.split():
+            eext = ext.split(':')
+            if len(eext) > 1:
+                if eext[0] == 'multilib':
+                    multilib_archs.append('ml' + eext[1])
+
+    d.setVar("MULTILIB_ARCHS", ' '.join(multilib_archs))
 }
 
 def splitfile(file, debugfile, debugsrcdir, d):
-- 
1.7.0.4

[MULTILIB_SUBTASK4 4/5] image.bbclass: Added variables for multilib support.

[Lianhao Lu]

1. Added MULTILIB_PACKAGE_INSTALL for multilib instances of packages to
be installed in the rootfs.

2. MULTILIBRE_ALLOW_REP contains the regular expression to match the
files allow to be replaced by the conflicting files.

3. MULTILIBRE_FORCE_SAME contains the regular expression to match the
files allow to be replaced only if the conflicting files are identical.

4. Added shell function multilib_sanity_check() to check whether the
overwring for multilib situation is allowed.

Signed-off-by: Lianhao Lu <lianhao.lu@intel.com>
---
 meta/classes/image.bbclass |   54 +++++++++++++++++++++++++++++++++++++++++++-
 1 files changed, 53 insertions(+), 1 deletions(-)

diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index 2469442..3aa3251 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -5,13 +5,15 @@ inherit imagetest-${IMAGETEST}
 
 LICENSE = "MIT"
 PACKAGES = ""
-RDEPENDS += "${IMAGE_INSTALL}"
+MULTILIB_IMAGE_INSTALL ?= ""
+RDEPENDS += "${IMAGE_INSTALL} ${MULTILIB_IMAGE_INSTALL}"
 
 INHIBIT_DEFAULT_DEPS = "1"
 
 # "export IMAGE_BASENAME" not supported at this time
 IMAGE_BASENAME[export] = "1"
 export PACKAGE_INSTALL ?= "${IMAGE_INSTALL}"
+export MULTILIB_PACKAGE_INSTALL ?= "${MULTILIB_IMAGE_INSTALL}"
 PACKAGE_INSTALL_ATTEMPTONLY ?= ""
 
 # Images are generally built explicitly, do not need to be part of world.
@@ -86,6 +88,7 @@ do_build[nostamp] = "1"
 fakeroot do_rootfs () {
 	#set -x
 	rm -rf ${IMAGE_ROOTFS}
+	rm -rf ${MULTILIB_TEMP_ROOTFS}
 	mkdir -p ${IMAGE_ROOTFS}
 	mkdir -p ${DEPLOY_DIR_IMAGE}
 
@@ -161,6 +164,55 @@ log_check() {
 	done
 }
 
+MULTILIBRE_ALLOW_REP =. "${base_bindir}|${base_sbindir}|${bindir}|${sbindir}|${libexecdir}|"
+MULTILIBRE_FORCE_SAME =. "${sysconfdir}|${datadir}|"
+MULTILIB_CHECK_FILE = "${WORKDIR}/multilib_check.py"
+MULTILIB_TEMP_ROOTFS = "${WORKDIR}/multilib"
+
+multilib_generate_python_file() {
+  cat >${MULTILIB_CHECK_FILE} <<EOF
+import sys, os, os.path
+import re,filecmp
+
+allow_rep=re.compile(re.sub("\|$","","${MULTILIBRE_ALLOW_REP}"))
+force_same=re.compile(re.sub("\|$","","${MULTILIBRE_FORCE_SAME}"))
+error_promt="Multilib check error:"
+
+files={}
+dirs=raw_input()
+for dir in dirs.split():
+  for root, subfolers, subfiles in os.walk(dir):
+    for file in subfiles:
+      item=os.path.join(root,file)
+      key=str(os.path.join("/",os.path.relpath(item,dir)))
+
+      valid=True;
+      if files.has_key(key):
+        #check whether files are the same
+        if force_same.match(key):
+          if not filecmp.cmp(files[key],item):
+             valid=False
+             print("%s %s is not the same as %s\n" % (error_promt, item, files[key]))
+             sys.exit(1)
+        #check whether the file is allow to replace
+        elif allow_rep.match(key):
+          valid=True
+        else: 
+          valid=False
+          print("%s duplicated files %s %s not allowed\n" % (error_promt, item, files[key]))
+          sys.exit(1)
+
+      #pass the check, add to list
+      if valid:
+        files[key]=item
+EOF
+}
+
+multilib_sanity_check() {
+  multilib_generate_python_file
+  echo $@ | python ${MULTILIB_CHECK_FILE}
+}
+
 # set '*' as the rootpassword so the images
 # can decide if they want it or not
 
-- 
1.7.0.4

[MULTILIB_SUBTASK4 5/5] package(rootfs)_ipk.bbclass: support multilib in opkg backend.

[Lianhao Lu]

Support install multiple multilib in opkg backend.

The installation is done in 3 phases.

Phase 1: install normal packages to IMAGE_ROOTFS.

Phase 2: install multilib packages under MULTILIB_TEMP_ROOTFS. Packages
belongs to the same multilib arch would be installed to a unique
directory.

Phase 3: check file confliction between IMAGE_ROOTFS and
MULTILIB_TEMP_ROOTFS, install multilib packages to IMAGE_ROOTFS only if
the sanity check passed.

Signed-off-by: Lianhao Lu <lianhao.lu@intel.com>
---
 meta/classes/package_ipk.bbclass |   51 ++++++++++++++++++++++++++++++++++++++
 meta/classes/rootfs_ipk.bbclass  |    3 ++
 2 files changed, 54 insertions(+), 0 deletions(-)

diff --git a/meta/classes/package_ipk.bbclass b/meta/classes/package_ipk.bbclass
index 4317028..7029a2e 100644
--- a/meta/classes/package_ipk.bbclass
+++ b/meta/classes/package_ipk.bbclass
@@ -61,6 +61,28 @@ python package_ipk_install () {
 		raise bb.build.FuncFailed
 }
 
+package_tryout_install_multilib_ipk() {
+	#try install multilib
+	multilib_tryout_dirs=""
+	for arch in ${MULTILIB_ARCHS}; do
+		local target_rootfs="${MULTILIB_TEMP_ROOTFS}/${arch}"
+		local ipkg_args="-f ${INSTALL_CONF_IPK} -o ${target_rootfs} --force_overwrite"
+		local selected_pkg=""
+		#strip the "ml" from package_arch
+		local pkgarch_prefix="${arch:2}-"
+		for pkg in "${INSTALL_PACKAGES_MULTILIB_IPK}"; do
+			if [ ${pkg:0:${#pkgarch_prefix}} == ${pkgarch_prefix} ]; then
+			    selected_pkg="${selected_pkg} ${pkg}"
+			fi
+		done
+		if [ ! -z "${selected_pkg}" ]; then
+			mkdir -p ${target_rootfs}/${opkglibdir}
+			opkg-cl ${ipkg_args} update
+			opkg-cl ${ipkg_args} install ${selected_pkg}
+			multilib_tryout_dirs="${multilib_tryout_dirs} ${target_rootfs}"
+		fi
+	done
+}
 #
 # install a bunch of packages using opkg
 # the following shell variables needs to be set before calling this func:
@@ -78,6 +100,7 @@ package_install_internal_ipk() {
 	local package_to_install="${INSTALL_PACKAGES_NORMAL_IPK}"
 	local package_attemptonly="${INSTALL_PACKAGES_ATTEMPTONLY_IPK}"
 	local package_lingusa="${INSTALL_PACKAGES_LINGUAS_IPK}"
+	local package_multilib="${INSTALL_PACKAGES_MULTILIB_IPK}"
 	local task="${INSTALL_TASK_IPK}"
 
 	mkdir -p ${target_rootfs}${localstatedir}/lib/opkg/
@@ -102,6 +125,14 @@ package_install_internal_ipk() {
 	if [ ! -z "${package_attemptonly}" ]; then
 		opkg-cl ${ipkg_args} install ${package_attemptonly} > "${WORKDIR}/temp/log.do_${task}_attemptonly.${PID}" || true
 	fi
+
+	package_tryout_install_multilib_ipk
+	#sanity check
+	multilib_sanity_check  ${target_rootfs} ${multilib_tryout_dirs}|| exit 1
+
+	if [ ! -z "${package_multilib}" ]; then
+		opkg-cl ${ipkg_args} install ${package_multilib}
+	fi
 }
 
 ipk_log_check() {
@@ -142,6 +173,11 @@ package_update_index_ipk () {
 		packagedirs="$packagedirs ${DEPLOY_DIR_IPK}/$arch ${DEPLOY_DIR_IPK}/$sdkarch-nativesdk"
 	done
 
+	multilib_archs="${MULTILIB_ARCHS}"
+	for arch in $multilib_archs; do
+		packagedirs="$packagedirs ${DEPLOY_DIR_IPK}/$arch"
+	done
+
 	for pkgdir in $packagedirs; do
 		if [ -e $pkgdir/ ]; then
 			touch $pkgdir/Packages
@@ -173,6 +209,13 @@ package_generate_ipkg_conf () {
 		        echo "src oe-$sdkarch$extension file:${DEPLOY_DIR_IPK}/$sdkarch$extension" >> ${IPKGCONF_SDK}
 		fi
 	done
+
+	multilib_archs="${MULTILIB_ARCHS}"
+	for arch in $multilib_archs; do
+		if [ -e ${DEPLOY_DIR_IPK}/$arch/Packages ] ; then
+		        echo "src oe-$arch file:${DEPLOY_DIR_IPK}/$arch" >> ${IPKGCONF_TARGET}
+		fi
+	done
 }
 
 package_generate_archlist () {
@@ -188,6 +231,14 @@ package_generate_archlist () {
 		echo "arch $sdkarch$extension $priority" >> ${IPKGCONF_SDK}
 		priority=$(expr $priority + 5)
 	done
+
+	multilib_archs="${MULTILIB_ARCHS}"
+	for arch in $multilib_archs; do
+		echo "arch $arch $priority" >> ${IPKGCONF_TARGET}
+		priority=$(expr $priority + 5)
+	done
+
+
 }
 
 python do_package_ipk () {
diff --git a/meta/classes/rootfs_ipk.bbclass b/meta/classes/rootfs_ipk.bbclass
index 3b4b4da..26d0676 100644
--- a/meta/classes/rootfs_ipk.bbclass
+++ b/meta/classes/rootfs_ipk.bbclass
@@ -20,6 +20,8 @@ OPKG_POSTPROCESS_COMMANDS = "ipk_insert_feed_uris"
 
 opkglibdir = "${localstatedir}/lib/opkg"
 
+MULTILIBRE_ALLOW_REP = "${opkglibdir}"
+
 fakeroot rootfs_ipk_do_rootfs () {
 	set -x
 
@@ -38,6 +40,7 @@ fakeroot rootfs_ipk_do_rootfs () {
 	export INSTALL_ROOTFS_IPK="${IMAGE_ROOTFS}"
 	export INSTALL_CONF_IPK="${IPKGCONF_TARGET}"
 	export INSTALL_PACKAGES_NORMAL_IPK="${PACKAGE_INSTALL}"
+	export INSTALL_PACKAGES_MULTILIB_IPK="${MULTILIB_PACKAGE_INSTALL}"
 
 	package_install_internal_ipk
 
-- 
1.7.0.4