[PATCH v2 1/3] kernel: allow kernel module and firmware installation with ${nonarch_base_libdir} != "/lib"

[PATCH v2 1/3] kernel: allow kernel module and firmware installation with ${nonarch_base_libdir} != "/lib"

[Matthias Schiffer]

Signed-off-by: Matthias Schiffer <mschiffer@universe-factory.net>
---
v2:
* also adjust comments
* use ${nonarch_base_libdir} instead of ${base_libdir}, modules and
  firmware are always expected under /lib, while ppc64 has ${base_libdir}
  = "/lib64".


 meta/classes/kernel-module-split.bbclass |  6 +++---
 meta/classes/kernel.bbclass              | 25 ++++++++++++++++++-------
 2 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/meta/classes/kernel-module-split.bbclass b/meta/classes/kernel-module-split.bbclass
index e1a70e6..dd31587 100644
--- a/meta/classes/kernel-module-split.bbclass
+++ b/meta/classes/kernel-module-split.bbclass
@@ -108,8 +108,8 @@ python split_kernel_module_packages () {
     def get_dependencies(file, pattern, format):
         # file no longer includes PKGD
         file = file.replace(d.getVar('PKGD', True) or '', '', 1)
-        # instead is prefixed with /lib/modules/${KERNEL_VERSION}
-        file = file.replace("/lib/modules/%s/" % d.getVar('KERNEL_VERSION', True) or '', '', 1)
+        # instead is prefixed with ${nonarch_base_libdir}/modules/${KERNEL_VERSION}
+        file = file.replace("%s/modules/%s/" % (d.getVar('nonarch_base_libdir', True), d.getVar('KERNEL_VERSION', True)), '', 1)
 
         if file in module_deps:
             dependencies = []
@@ -186,7 +186,7 @@ python split_kernel_module_packages () {
     postinst = d.getVar('pkg_postinst_modules', True)
     postrm = d.getVar('pkg_postrm_modules', True)
 
-    modules = do_split_packages(d, root='/lib/modules', file_regex=module_regex, output_pattern=module_pattern, description='%s kernel module', postinst=postinst, postrm=postrm, recursive=True, hook=frob_metadata, extra_depends='kernel-%s' % (d.getVar("KERNEL_VERSION", True)))
+    modules = do_split_packages(d, root='%s/modules' % d.getVar('nonarch_base_libdir', True), file_regex=module_regex, output_pattern=module_pattern, description='%s kernel module', postinst=postinst, postrm=postrm, recursive=True, hook=frob_metadata, extra_depends='kernel-%s' % (d.getVar("KERNEL_VERSION", True)))
     if modules:
         metapkg = d.getVar('KERNEL_MODULES_META_PACKAGE', True)
         d.appendVar('RDEPENDS_' + metapkg, ' '+' '.join(modules))
diff --git a/meta/classes/kernel.bbclass b/meta/classes/kernel.bbclass
index 4ce1611..3794ee0 100644
--- a/meta/classes/kernel.bbclass
+++ b/meta/classes/kernel.bbclass
@@ -242,6 +242,11 @@ kernel_do_install() {
 		rm "${D}/lib/modules/${KERNEL_VERSION}/source"
 		# If the kernel/ directory is empty remove it to prevent QA issues
 		rmdir --ignore-fail-on-non-empty "${D}/lib/modules/${KERNEL_VERSION}/kernel"
+
+		if [ "${nonarch_base_libdir}" != "/lib" ]; then
+			install -d "${D}${nonarch_base_libdir}"
+			mv "${D}/lib/modules" "${D}${nonarch_base_libdir}"
+		fi
 	else
 		bbnote "no modules to install"
 	fi
@@ -258,6 +263,12 @@ kernel_do_install() {
 	[ -e Module.symvers ] && install -m 0644 Module.symvers ${D}/boot/Module.symvers-${KERNEL_VERSION}
 	install -d ${D}${sysconfdir}/modules-load.d
 	install -d ${D}${sysconfdir}/modprobe.d
+
+	if [ "${nonarch_base_libdir}" != "/lib" ]; then
+		install -d "${D}${nonarch_base_libdir}"
+		mv "${D}/lib/firmware" "${D}${nonarch_base_libdir}"
+		rmdir --ignore-fail-on-non-empty "${D}/lib"
+	fi
 }
 do_install[prefuncs] += "package_get_auto_pr"
 
@@ -364,9 +375,9 @@ EXPORT_FUNCTIONS do_compile do_install do_configure
 # kernel-image becomes kernel-image-${KERNEL_VERISON}
 PACKAGES = "kernel kernel-base kernel-vmlinux kernel-image kernel-dev kernel-modules"
 FILES_${PN} = ""
-FILES_kernel-base = "/lib/modules/${KERNEL_VERSION}/modules.order /lib/modules/${KERNEL_VERSION}/modules.builtin"
+FILES_kernel-base = "${nonarch_base_libdir}/modules/${KERNEL_VERSION}/modules.order ${nonarch_base_libdir}/modules/${KERNEL_VERSION}/modules.builtin"
 FILES_kernel-image = "/boot/${KERNEL_IMAGETYPE}*"
-FILES_kernel-dev = "/boot/System.map* /boot/Module.symvers* /boot/config* ${KERNEL_SRC_PATH} /lib/modules/${KERNEL_VERSION}/build"
+FILES_kernel-dev = "/boot/System.map* /boot/Module.symvers* /boot/config* ${KERNEL_SRC_PATH} ${nonarch_base_libdir}/modules/${KERNEL_VERSION}/build"
 FILES_kernel-vmlinux = "/boot/vmlinux*"
 FILES_kernel-modules = ""
 RDEPENDS_kernel = "kernel-base"
@@ -384,8 +395,8 @@ ALLOW_EMPTY_kernel-modules = "1"
 DESCRIPTION_kernel-modules = "Kernel modules meta package"
 
 pkg_postinst_kernel-base () {
-	if [ ! -e "$D/lib/modules/${KERNEL_VERSION}" ]; then
-		mkdir -p $D/lib/modules/${KERNEL_VERSION}
+	if [ ! -e "$D${nonarch_base_libdir}/modules/${KERNEL_VERSION}" ]; then
+		mkdir -p $D${nonarch_base_libdir}/modules/${KERNEL_VERSION}
 	fi
 	if [ -n "$D" ]; then
 		depmodwrapper -a -b $D ${KERNEL_VERSION}
@@ -405,7 +416,7 @@ pkg_postrm_kernel-image () {
 PACKAGESPLITFUNCS_prepend = "split_kernel_packages "
 
 python split_kernel_packages () {
-    do_split_packages(d, root='/lib/firmware', file_regex='^(.*)\.(bin|fw|cis|dsp)$', output_pattern='kernel-firmware-%s', description='Firmware for %s', recursive=True, extra_depends='')
+    do_split_packages(d, root='%s/firmware' % d.getVar('nonarch_base_libdir', True), file_regex='^(.*)\.(bin|fw|cis|dsp)$', output_pattern='kernel-firmware-%s', description='Firmware for %s', recursive=True, extra_depends='')
 }
 
 do_strip() {
@@ -469,8 +480,8 @@ MODULE_TARBALL_DEPLOY ?= "1"
 kernel_do_deploy() {
 	install -m 0644 ${KERNEL_OUTPUT} ${DEPLOYDIR}/${KERNEL_IMAGE_BASE_NAME}.bin
 	if [ ${MODULE_TARBALL_DEPLOY} = "1" ] && (grep -q -i -e '^CONFIG_MODULES=y$' .config); then
-		mkdir -p ${D}/lib
-		tar -cvzf ${DEPLOYDIR}/${MODULE_TARBALL_BASE_NAME} -C ${D} lib
+		mkdir -p ${D}${nonarch_base_libdir}
+		tar -cvzf ${DEPLOYDIR}/${MODULE_TARBALL_BASE_NAME} -C ${D} ${@d.getVar('nonarch_base_libdir', True)[1:]}
 		ln -sf ${MODULE_TARBALL_BASE_NAME} ${DEPLOYDIR}/${MODULE_TARBALL_SYMLINK_NAME}
 	fi
 
-- 
2.6.4

[PATCH v2 2/3] base-files: create ${base_bindir} etc. instead of /bin, /sbin and /lib

[Matthias Schiffer]

These directories conflict with the symlinks created for merged-usr setups.

Signed-off-by: Matthias Schiffer <mschiffer@universe-factory.net>
---
v2: create both ${base_libdir} and ${nonarch_base_libdir}

 meta/recipes-core/base-files/base-files_3.0.14.bb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/meta/recipes-core/base-files/base-files_3.0.14.bb b/meta/recipes-core/base-files/base-files_3.0.14.bb
index b71d5c5..2af7ecd 100644
--- a/meta/recipes-core/base-files/base-files_3.0.14.bb
+++ b/meta/recipes-core/base-files/base-files_3.0.14.bb
@@ -33,8 +33,8 @@ INHIBIT_DEFAULT_DEPS = "1"
 docdir_append = "/${P}"
 dirs1777 = "/tmp ${localstatedir}/volatile/tmp"
 dirs2775 = ""
-dirs755 = "/bin /boot /dev ${sysconfdir} ${sysconfdir}/default \
-           ${sysconfdir}/skel /lib /mnt /proc ${ROOT_HOME} /run /sbin \
+dirs755 = "${base_bindir} /boot /dev ${sysconfdir} ${sysconfdir}/default \
+           ${sysconfdir}/skel ${base_libdir} ${nonarch_base_libdir} /mnt /proc ${ROOT_HOME} /run ${base_sbindir} \
            ${prefix} ${bindir} ${docdir} /usr/games ${includedir} \
            ${libdir} ${sbindir} ${datadir} \
            ${datadir}/common-licenses ${datadir}/dict ${infodir} \
-- 
2.6.4

[PATCH v2 3/3] base-files: create typical merged /usr symlinks if the "merged-usr" distro feature is set

[Matthias Schiffer]

The typical symlinks are:

* /bin -> usr/bin
* /sbin -> usr/sbin
* /lib -> usr/lib
* /usr/sbin -> bin

It is checked that only sane symlinks are created, so a half-merged /usr
(e.g, merging bin and sbin, but not lib) is feasible as well.

The following options in a distro configuration create a full Fedora-style
merged /usr:

  DISTRO_FEATURES_DEFAULT += "merged-usr"
  base_bindir = "${exec_prefix}/bin"
  base_sbindir = "${exec_prefix}/bin"
  sbindir = "${exec_prefix}/bin"
  base_libdir = "${exec_prefix}/${baselib}"
  nonarch_base_libdir = "${exec_prefix}/lib"

[YOCTO #7040]

Signed-off-by: Matthias Schiffer <mschiffer@universe-factory.net>
---
v2:
* add the merged-usr symlinks in a separate task, so do_install can be
  overridden independently of the merged-usr feature.
* add PACKAGECONFIG for merged-usr, so the package is rebuilt when the
  distro feature is changed.


 meta/recipes-core/base-files/base-files_3.0.14.bb | 36 +++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/meta/recipes-core/base-files/base-files_3.0.14.bb b/meta/recipes-core/base-files/base-files_3.0.14.bb
index 2af7ecd..21a740f 100644
--- a/meta/recipes-core/base-files/base-files_3.0.14.bb
+++ b/meta/recipes-core/base-files/base-files_3.0.14.bb
@@ -30,6 +30,13 @@ S = "${WORKDIR}"
 
 INHIBIT_DEFAULT_DEPS = "1"
 
+PACKAGECONFIG ??= " \
+                   ${@bb.utils.contains('DISTRO_FEATURES', 'merged-usr','merged-usr', '', d)} \
+"
+
+PACKAGECONFIG[merged-usr] = ",,"
+
+
 docdir_append = "/${P}"
 dirs1777 = "/tmp ${localstatedir}/volatile/tmp"
 dirs2775 = ""
@@ -136,6 +143,35 @@ do_install () {
 	ln -sf /proc/mounts ${D}${sysconfdir}/mtab
 }
 
+do_install_merged_usr () {
+	if ${@bb.utils.contains('PACKAGECONFIG','merged-usr','false','true',d)}; then
+		return
+	fi
+
+	if [ "${base_bindir}" != "/bin" ]; then
+		ln -snf "${@d.getVar('base_bindir', True)[1:]}" ${D}/bin
+	fi
+
+	if [ "${base_sbindir}" != "/sbin"  ]; then
+		ln -snf "${@d.getVar('base_sbindir', True)[1:]}" ${D}/sbin
+	fi
+
+	if [ "${nonarch_base_libdir}" != "/lib" ]; then
+		ln -snf "${@d.getVar('nonarch_base_libdir', True)[1:]}" ${D}/lib
+	fi
+
+	if [ "${base_libdir}" != "/${baselib}" ]; then
+		ln -snf "${@d.getVar('base_libdir', True)[1:]}" ${D}/${baselib}
+	fi
+
+	if [ "${bindir}" = "${exec_prefix}/bin" -a "${sbindir}" = "${exec_prefix}/bin" ]; then
+		ln -snf bin ${D}${exec_prefix}/sbin
+	fi
+}
+
+addtask install_merged_usr after do_install before do_package do_populate_sysroot
+
+
 DISTRO_VERSION[vardepsexclude] += "DATE"
 do_install_basefilesissue () {
 	if [ "${hostname}" ]; then
-- 
2.6.4

Re: [PATCH v2 3/3] base-files: create typical merged /usr symlinks if the "merged-usr" distro feature is set

[Phil Blundell]

On Sun, 2016-01-03 at 00:53 +0100, Matthias Schiffer wrote:
> +PACKAGECONFIG ??= " \
> +                   ${@bb.utils.contains('DISTRO_FEATURES', 'merged-usr','merged-usr', '', d)} \

Why is this DISTRO_FEATURE (or the corresponding PACKAGECONFIG)
required?  Surely it ought to suffice to simply set ${base_bindir} etc,
and everything else should follow from that.  This, albeit the other way
around, is what micro has been doing for years.

p.

Re: [PATCH v2 3/3] base-files: create typical merged /usr symlinks if the "merged-usr" distro feature is set

[Matthias Schiffer]

[-- Attachment #1: Type: text/plain, Size: 1095 bytes --]

On 01/03/2016 02:01 PM, Phil Blundell wrote:
> On Sun, 2016-01-03 at 00:53 +0100, Matthias Schiffer wrote:
>> +PACKAGECONFIG ??= " \
>> +                   ${@bb.utils.contains('DISTRO_FEATURES', 'merged-usr','merged-usr', '', d)} \
> 
> Why is this DISTRO_FEATURE (or the corresponding PACKAGECONFIG)
> required?  Surely it ought to suffice to simply set ${base_bindir} etc,
> and everything else should follow from that.  This, albeit the other way
> around, is what micro has been doing for years.
> 
> p.
> 
> 

Basically, I didn't want to break existing configurations. There might
be unusual distros which don't want these symlinks (or want different
ones). Of course, the alternative would be to force such distros to
override the bases-files recipe in a .bbappend.

I don't know of any OE-based distros with such unusual setups, but
Debian is an example: they basically set base_libdir to /lib/$TARGET,
but don't symlink /lib64 to this on 64bit targets. Instead, /lib64 is
empty except of a symlink to the dynamic linker which is expected in /lib64.

Matthias


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 836 bytes --]

Re: [PATCH v2 3/3] base-files: create typical merged /usr symlinks if the "merged-usr" distro feature is set

[Mark Hatle]

On 1/2/16 5:53 PM, Matthias Schiffer wrote:
> The typical symlinks are:
> 
> * /bin -> usr/bin
> * /sbin -> usr/sbin
> * /lib -> usr/lib
> * /usr/sbin -> bin
> 
> It is checked that only sane symlinks are created, so a half-merged /usr
> (e.g, merging bin and sbin, but not lib) is feasible as well.
> 
> The following options in a distro configuration create a full Fedora-style
> merged /usr:
> 
>   DISTRO_FEATURES_DEFAULT += "merged-usr"
>   base_bindir = "${exec_prefix}/bin"
>   base_sbindir = "${exec_prefix}/bin"
>   sbindir = "${exec_prefix}/bin"
>   base_libdir = "${exec_prefix}/${baselib}"
>   nonarch_base_libdir = "${exec_prefix}/lib"
> 
> [YOCTO #7040]
> 
> Signed-off-by: Matthias Schiffer <mschiffer@universe-factory.net>
> ---
> v2:
> * add the merged-usr symlinks in a separate task, so do_install can be
>   overridden independently of the merged-usr feature.
> * add PACKAGECONFIG for merged-usr, so the package is rebuilt when the
>   distro feature is changed.
> 
> 
>  meta/recipes-core/base-files/base-files_3.0.14.bb | 36 +++++++++++++++++++++++
>  1 file changed, 36 insertions(+)
> 
> diff --git a/meta/recipes-core/base-files/base-files_3.0.14.bb b/meta/recipes-core/base-files/base-files_3.0.14.bb
> index 2af7ecd..21a740f 100644
> --- a/meta/recipes-core/base-files/base-files_3.0.14.bb
> +++ b/meta/recipes-core/base-files/base-files_3.0.14.bb
> @@ -30,6 +30,13 @@ S = "${WORKDIR}"
>  
>  INHIBIT_DEFAULT_DEPS = "1"
>  
> +PACKAGECONFIG ??= " \
> +                   ${@bb.utils.contains('DISTRO_FEATURES', 'merged-usr','merged-usr', '', d)} \
> +"
> +
> +PACKAGECONFIG[merged-usr] = ",,"
> +
> +
>  docdir_append = "/${P}"
>  dirs1777 = "/tmp ${localstatedir}/volatile/tmp"
>  dirs2775 = ""
> @@ -136,6 +143,35 @@ do_install () {
>  	ln -sf /proc/mounts ${D}${sysconfdir}/mtab
>  }
>  
> +do_install_merged_usr () {
> +	if ${@bb.utils.contains('PACKAGECONFIG','merged-usr','false','true',d)}; then
> +		return
> +	fi
> +
> +	if [ "${base_bindir}" != "/bin" ]; then
> +		ln -snf "${@d.getVar('base_bindir', True)[1:]}" ${D}/bin
> +	fi
> +
> +	if [ "${base_sbindir}" != "/sbin"  ]; then
> +		ln -snf "${@d.getVar('base_sbindir', True)[1:]}" ${D}/sbin
> +	fi
> +
> +	if [ "${nonarch_base_libdir}" != "/lib" ]; then
> +		ln -snf "${@d.getVar('nonarch_base_libdir', True)[1:]}" ${D}/lib
> +	fi
> +
> +	if [ "${base_libdir}" != "/${baselib}" ]; then
> +		ln -snf "${@d.getVar('base_libdir', True)[1:]}" ${D}/${baselib}
> +	fi
> +
> +	if [ "${bindir}" = "${exec_prefix}/bin" -a "${sbindir}" = "${exec_prefix}/bin" ]; then
> +		ln -snf bin ${D}${exec_prefix}/sbin
> +	fi
> +}

The stuff above really seems to me that it should be done by the fs-perms setup.
 Is the real problem here that people don't understand the fs-perms and how to
configure it for these alternative filesystem configurations?

(or if the code in the packaging isn't working, we should fix it.)

--Mark

> +
> +addtask install_merged_usr after do_install before do_package do_populate_sysroot
> +
> +
>  DISTRO_VERSION[vardepsexclude] += "DATE"
>  do_install_basefilesissue () {
>  	if [ "${hostname}" ]; then
> 

Re: [PATCH v2 2/3] base-files: create ${base_bindir} etc. instead of /bin, /sbin and /lib

[Mark Hatle]

On 1/2/16 5:53 PM, Matthias Schiffer wrote:
> These directories conflict with the symlinks created for merged-usr setups.
> 
> Signed-off-by: Matthias Schiffer <mschiffer@universe-factory.net>
> ---
> v2: create both ${base_libdir} and ${nonarch_base_libdir}
> 
>  meta/recipes-core/base-files/base-files_3.0.14.bb | 4 ++--
>  1 file changed, 2 insertions(+), 2 deletions(-)
> 
> diff --git a/meta/recipes-core/base-files/base-files_3.0.14.bb b/meta/recipes-core/base-files/base-files_3.0.14.bb
> index b71d5c5..2af7ecd 100644
> --- a/meta/recipes-core/base-files/base-files_3.0.14.bb
> +++ b/meta/recipes-core/base-files/base-files_3.0.14.bb
> @@ -33,8 +33,8 @@ INHIBIT_DEFAULT_DEPS = "1"
>  docdir_append = "/${P}"
>  dirs1777 = "/tmp ${localstatedir}/volatile/tmp"
>  dirs2775 = ""
> -dirs755 = "/bin /boot /dev ${sysconfdir} ${sysconfdir}/default \
> -           ${sysconfdir}/skel /lib /mnt /proc ${ROOT_HOME} /run /sbin \
> +dirs755 = "${base_bindir} /boot /dev ${sysconfdir} ${sysconfdir}/default \
> +           ${sysconfdir}/skel ${base_libdir} ${nonarch_base_libdir} /mnt /proc ${ROOT_HOME} /run ${base_sbindir} \
>             ${prefix} ${bindir} ${docdir} /usr/games ${includedir} \
>             ${libdir} ${sbindir} ${datadir} \
>             ${datadir}/common-licenses ${datadir}/dict ${infodir} \
> 

I agree this new set looks correct.. but I'd like to see the corresponding
change to fs-perms that sets the permissions and such for each of these new
locations.

--Mark

Re: [PATCH v2 2/3] base-files: create ${base_bindir} etc. instead of /bin, /sbin and /lib

[Matthias Schiffer]

[-- Attachment #1: Type: text/plain, Size: 1913 bytes --]

On 01/04/2016 11:59 PM, Mark Hatle wrote:
> On 1/2/16 5:53 PM, Matthias Schiffer wrote:
>> These directories conflict with the symlinks created for merged-usr setups.
>>
>> Signed-off-by: Matthias Schiffer <mschiffer@universe-factory.net>
>> ---
>> v2: create both ${base_libdir} and ${nonarch_base_libdir}
>>
>>  meta/recipes-core/base-files/base-files_3.0.14.bb | 4 ++--
>>  1 file changed, 2 insertions(+), 2 deletions(-)
>>
>> diff --git a/meta/recipes-core/base-files/base-files_3.0.14.bb b/meta/recipes-core/base-files/base-files_3.0.14.bb
>> index b71d5c5..2af7ecd 100644
>> --- a/meta/recipes-core/base-files/base-files_3.0.14.bb
>> +++ b/meta/recipes-core/base-files/base-files_3.0.14.bb
>> @@ -33,8 +33,8 @@ INHIBIT_DEFAULT_DEPS = "1"
>>  docdir_append = "/${P}"
>>  dirs1777 = "/tmp ${localstatedir}/volatile/tmp"
>>  dirs2775 = ""
>> -dirs755 = "/bin /boot /dev ${sysconfdir} ${sysconfdir}/default \
>> -           ${sysconfdir}/skel /lib /mnt /proc ${ROOT_HOME} /run /sbin \
>> +dirs755 = "${base_bindir} /boot /dev ${sysconfdir} ${sysconfdir}/default \
>> +           ${sysconfdir}/skel ${base_libdir} ${nonarch_base_libdir} /mnt /proc ${ROOT_HOME} /run ${base_sbindir} \
>>             ${prefix} ${bindir} ${docdir} /usr/games ${includedir} \
>>             ${libdir} ${sbindir} ${datadir} \
>>             ${datadir}/common-licenses ${datadir}/dict ${infodir} \
>>
> 
> I agree this new set looks correct.. but I'd like to see the corresponding
> change to fs-perms that sets the permissions and such for each of these new
> locations.
> 
> --Mark
> 

The current meta/files/fs-perms.txt doesn't mention any of /bin, /sbin
or /lib, and seemingly never did. I guess they could be added, but that
could also be done in a separate patch (and as I mentioned in another
mail, I'm not exactly a fan of the fs-perms concept, so I don't really
care.)

Matthias


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 836 bytes --]

Re: [PATCH v2 2/3] base-files: create ${base_bindir} etc. instead of /bin, /sbin and /lib

[Mark Hatle]

On 1/4/16 5:46 PM, Matthias Schiffer wrote:
> On 01/04/2016 11:59 PM, Mark Hatle wrote:
>> On 1/2/16 5:53 PM, Matthias Schiffer wrote:
>>> These directories conflict with the symlinks created for merged-usr setups.
>>>
>>> Signed-off-by: Matthias Schiffer <mschiffer@universe-factory.net>
>>> ---
>>> v2: create both ${base_libdir} and ${nonarch_base_libdir}
>>>
>>>  meta/recipes-core/base-files/base-files_3.0.14.bb | 4 ++--
>>>  1 file changed, 2 insertions(+), 2 deletions(-)
>>>
>>> diff --git a/meta/recipes-core/base-files/base-files_3.0.14.bb b/meta/recipes-core/base-files/base-files_3.0.14.bb
>>> index b71d5c5..2af7ecd 100644
>>> --- a/meta/recipes-core/base-files/base-files_3.0.14.bb
>>> +++ b/meta/recipes-core/base-files/base-files_3.0.14.bb
>>> @@ -33,8 +33,8 @@ INHIBIT_DEFAULT_DEPS = "1"
>>>  docdir_append = "/${P}"
>>>  dirs1777 = "/tmp ${localstatedir}/volatile/tmp"
>>>  dirs2775 = ""
>>> -dirs755 = "/bin /boot /dev ${sysconfdir} ${sysconfdir}/default \
>>> -           ${sysconfdir}/skel /lib /mnt /proc ${ROOT_HOME} /run /sbin \
>>> +dirs755 = "${base_bindir} /boot /dev ${sysconfdir} ${sysconfdir}/default \
>>> +           ${sysconfdir}/skel ${base_libdir} ${nonarch_base_libdir} /mnt /proc ${ROOT_HOME} /run ${base_sbindir} \
>>>             ${prefix} ${bindir} ${docdir} /usr/games ${includedir} \
>>>             ${libdir} ${sbindir} ${datadir} \
>>>             ${datadir}/common-licenses ${datadir}/dict ${infodir} \
>>>
>>
>> I agree this new set looks correct.. but I'd like to see the corresponding
>> change to fs-perms that sets the permissions and such for each of these new
>> locations.
>>
>> --Mark
>>
> 
> The current meta/files/fs-perms.txt doesn't mention any of /bin, /sbin
> or /lib, and seemingly never did. I guess they could be added, but that
> could also be done in a separate patch (and as I mentioned in another
> mail, I'm not exactly a fan of the fs-perms concept, so I don't really
> care.)

Actually it does:

# Note: all standard config directories are automatically assigned "0755 root
root false - - -"

Actual definition is buried in the code itself, meta/classes/package.bbclass:

    # By default all of the standard directories specified in
    # bitbake.conf will get 0755 root:root.
    target_path_vars = [    'base_prefix',
                'prefix',
                'exec_prefix',
                'base_bindir',
                'base_sbindir',
                'base_libdir',
                'datadir',
                'sysconfdir',
                'servicedir',
                'sharedstatedir',
                'localstatedir',
                'infodir',
                'mandir',
                'docdir',
                'bindir',
                'sbindir',
                'libexecdir',
                'libdir',
                'includedir',
                'oldincludedir' ]

The problem is people have added 'additional' standard directories since that
time and have NOT updated the table.

So likely the following needs to be added (based on the current bitbake.conf):

nonarch_base_libdir
systemd_unitdir
systemd_system_unitdir
nonarch_libdir
systemd_user_unitdir


As mentioned before.. the problem being solved is: If you don't have the table
of system directories and permission.. and you build a package that uses a
standard system directory.. what owner/group/permission does that directory get?

Answer? depends on the package, package manager and other misc things.. this is
a really really bad case on non-deterministic behavior.  If we run through a
table of operations (this internal table and the one in the fs-perms.txt and
other files, if added) then we can synchronize -all- packages in a build with
the configured system setup.

Note, with the way this was designed.. if /bin should NOT be a directory, the
fs-perms.txt (or additional files provided by a configuration) can simply
declare it as a symlink and the original entry goes away.

A distribution definition is a combination of the distro config file, matching
fs-perms settings, and other associated files.  People miss the fs-perms, I
don't know if they don't realize it's there and why it's there or if they assume
it's a one size fits all default configuration.  (The default does fit many
configurations, but not all....)

--Mark


> Matthias
> 

Re: [PATCH v2 3/3] base-files: create typical merged /usr symlinks if the "merged-usr" distro feature is set

[Phil Blundell]

On Mon, 2016-01-04 at 23:38 +0100, Matthias Schiffer wrote:
> Of course, the alternative would be to force such distros to
> override the bases-files recipe in a .bbappend.

I don't think this would be an especially big deal.  The whole contents
of base-files is already fairly distro-specific and almost any
non-trivial distro is probably going to be either adjusting or replacing
that recipe anyway.

Alternatively, one could argue that these symlinks have nothing much to
do with the rest of base-files and they should go in a completely
separate recipe which the distro could add to IMAGE_INSTALL if it
wished.  That might be even better.

p.

Re: [PATCH v2 2/3] base-files: create ${base_bindir} etc. instead of /bin, /sbin and /lib

[Matthias Schiffer]

[-- Attachment #1: Type: text/plain, Size: 5949 bytes --]

On 01/05/2016 01:16 AM, Mark Hatle wrote:
> On 1/4/16 5:46 PM, Matthias Schiffer wrote:
>> On 01/04/2016 11:59 PM, Mark Hatle wrote:
>>> On 1/2/16 5:53 PM, Matthias Schiffer wrote:
>>>> These directories conflict with the symlinks created for merged-usr setups.
>>>>
>>>> Signed-off-by: Matthias Schiffer <mschiffer@universe-factory.net>
>>>> ---
>>>> v2: create both ${base_libdir} and ${nonarch_base_libdir}
>>>>
>>>>  meta/recipes-core/base-files/base-files_3.0.14.bb | 4 ++--
>>>>  1 file changed, 2 insertions(+), 2 deletions(-)
>>>>
>>>> diff --git a/meta/recipes-core/base-files/base-files_3.0.14.bb b/meta/recipes-core/base-files/base-files_3.0.14.bb
>>>> index b71d5c5..2af7ecd 100644
>>>> --- a/meta/recipes-core/base-files/base-files_3.0.14.bb
>>>> +++ b/meta/recipes-core/base-files/base-files_3.0.14.bb
>>>> @@ -33,8 +33,8 @@ INHIBIT_DEFAULT_DEPS = "1"
>>>>  docdir_append = "/${P}"
>>>>  dirs1777 = "/tmp ${localstatedir}/volatile/tmp"
>>>>  dirs2775 = ""
>>>> -dirs755 = "/bin /boot /dev ${sysconfdir} ${sysconfdir}/default \
>>>> -           ${sysconfdir}/skel /lib /mnt /proc ${ROOT_HOME} /run /sbin \
>>>> +dirs755 = "${base_bindir} /boot /dev ${sysconfdir} ${sysconfdir}/default \
>>>> +           ${sysconfdir}/skel ${base_libdir} ${nonarch_base_libdir} /mnt /proc ${ROOT_HOME} /run ${base_sbindir} \
>>>>             ${prefix} ${bindir} ${docdir} /usr/games ${includedir} \
>>>>             ${libdir} ${sbindir} ${datadir} \
>>>>             ${datadir}/common-licenses ${datadir}/dict ${infodir} \
>>>>
>>>
>>> I agree this new set looks correct.. but I'd like to see the corresponding
>>> change to fs-perms that sets the permissions and such for each of these new
>>> locations.
>>>
>>> --Mark
>>>
>>
>> The current meta/files/fs-perms.txt doesn't mention any of /bin, /sbin
>> or /lib, and seemingly never did. I guess they could be added, but that
>> could also be done in a separate patch (and as I mentioned in another
>> mail, I'm not exactly a fan of the fs-perms concept, so I don't really
>> care.)
> 
> Actually it does:
> 
> # Note: all standard config directories are automatically assigned "0755 root
> root false - - -"
> 
> Actual definition is buried in the code itself, meta/classes/package.bbclass:
> 
>     # By default all of the standard directories specified in
>     # bitbake.conf will get 0755 root:root.
>     target_path_vars = [    'base_prefix',
>                 'prefix',
>                 'exec_prefix',
>                 'base_bindir',
>                 'base_sbindir',
>                 'base_libdir',
>                 'datadir',
>                 'sysconfdir',
>                 'servicedir',
>                 'sharedstatedir',
>                 'localstatedir',
>                 'infodir',
>                 'mandir',
>                 'docdir',
>                 'bindir',
>                 'sbindir',
>                 'libexecdir',
>                 'libdir',
>                 'includedir',
>                 'oldincludedir' ]
> 
> The problem is people have added 'additional' standard directories since that
> time and have NOT updated the table.
> 
> So likely the following needs to be added (based on the current bitbake.conf):
> 
> nonarch_base_libdir
> systemd_unitdir
> systemd_system_unitdir
> nonarch_libdir
> systemd_user_unitdir
> 
> 
> As mentioned before.. the problem being solved is: If you don't have the table
> of system directories and permission.. and you build a package that uses a
> standard system directory.. what owner/group/permission does that directory get?
> 
> Answer? depends on the package, package manager and other misc things.. this is
> a really really bad case on non-deterministic behavior.  If we run through a
> table of operations (this internal table and the one in the fs-perms.txt and
> other files, if added) then we can synchronize -all- packages in a build with
> the configured system setup.
> 
> Note, with the way this was designed.. if /bin should NOT be a directory, the
> fs-perms.txt (or additional files provided by a configuration) can simply
> declare it as a symlink and the original entry goes away.
> 
> A distribution definition is a combination of the distro config file, matching
> fs-perms settings, and other associated files.  People miss the fs-perms, I
> don't know if they don't realize it's there and why it's there or if they assume
> it's a one size fits all default configuration.  (The default does fit many
> configurations, but not all....)
> 
> --Mark
> 
> 
>> Matthias
>>
> 

Okay, I've thought a bit more about this now.

* I'm fine with fs-perms.txt fixing up permissions, that is actually a
good idea (I haven't yet researched though if any non-OE distros have a
similar feature.)


* I agree with you that having a base package like base-files create all
symlinks defined in fs-perms.txt would make sense.

I'd like to send a v3 of my patch which does that, thus dropping the
distro feature / package config again. If such an approach was rejected
a few years ago, I'd like to get a bit more input on this first, I
couldn't find that particular discussion in the archives.

(I'd also like to add some documentation about fs-perms.txt as comments
in the base-files recipe, so people who look into adjusting the
base-files package will find it.)


* I stand by my opinion that moving files automatically is a bad idea,
and the FILES issue mentioned in the other thread further backs my point
(besides that issue, I've mentioned relative symlinks and relative paths
in general as problems that can break packages when their files are moved).

My proposal would be to change the fs-perms.txt handling to only check
if nothing conflicts with default symlinks, but not to create or move
anything. This is another point I'd like to get more input on.


Matthias


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 836 bytes --]

Re: [PATCH v2 2/3] base-files: create ${base_bindir} etc. instead of /bin, /sbin and /lib

[Roman Khimov]

В письме от 10 января 2016 18:13:03 пользователь Matthias Schiffer написал:
> * I stand by my opinion that moving files automatically is a bad idea,

But that's exactly what can make many of /usr merge changes automatically. 
Take the kernel as an example, you just don't need the first two hunks of the 
kernel.bbclass patch, fs-perms fixes that for you if it's told that /lib is a 
symlink to 'usr/lib'.

> and the FILES issue mentioned in the other thread further backs my point

IMO, FILES just shouldn't use hard-coded /lib or any other hard-coded path 
like that.

-- 
 http://roman.khimov.ru
mailto: roman@khimov.ru
gpg --keyserver hkp://subkeys.pgp.net --recv-keys 0xE5E055C3

Re: [PATCH v2 2/3] base-files: create ${base_bindir} etc. instead of /bin, /sbin and /lib

[Matthias Schiffer]

[-- Attachment #1: Type: text/plain, Size: 1067 bytes --]

On 01/10/2016 06:52 PM, Roman Khimov wrote:
> В письме от 10 января 2016 18:13:03 пользователь Matthias Schiffer написал:
>> * I stand by my opinion that moving files automatically is a bad idea,
> 
> But that's exactly what can make many of /usr merge changes automatically. 
> Take the kernel as an example, you just don't need the first two hunks of the 
> kernel.bbclass patch, fs-perms fixes that for you if it's told that /lib is a 
> symlink to 'usr/lib'.

I can see that it makes things simpler, but I'm more interested in a
correct solution than a simple one.

> 
>> and the FILES issue mentioned in the other thread further backs my point
> 
> IMO, FILES just shouldn't use hard-coded /lib or any other hard-coded path 
> like that.
> 

But if FILES depends on ${base_libdir} etc., the rest of the package
should also be adjusted to install all its files into ${base_libdir}
instead of /lib. Installing files to /lib, but putting ${base_libdir}
into FILES seems extremely broken to me.

Matthias


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 836 bytes --]

Re: [PATCH v2 2/3] base-files: create ${base_bindir} etc. instead of /bin, /sbin and /lib

[Phil Blundell]

On Sun, 2016-01-10 at 20:52 +0300, Roman Khimov wrote:
> IMO, FILES just shouldn't use hard-coded /lib or any other hard-coded path 
> like that.

Well, right, but that doesn't really address the issue.  Even if FILES
is defined in terms of ${base_libdir} or whatever, it's still not going
to match if fixup_perms() has shuffled things around in some random way.
I don't think there is any reliable way for fixup_perms() to find and
patch all the variables that might refer to paths that it has decided to
adjust.

Conversely, if FILES is defined in terms of ${base_libdir} and
do_install() puts the files there, you can change the value of
${base_libdir} to whatever you like and everything will "just work"
without the need for fixup_perms() to move anything.

As I mentioned somewhere else in this thread, micro has been using a
merged /usr for several years (but done the other way around, so
everything is installed into /bin and suchlike) and this has worked just
fine without the need for any fsperms magic.  I can't see any obvious
reason why the new-style /usr merge is fundamentally more difficult.

p.

Re: [PATCH v2 2/3] base-files: create ${base_bindir} etc. instead of /bin, /sbin and /lib

[Mark Hatle]

On 1/10/16 11:13 AM, Matthias Schiffer wrote:
(Trimming to just this comment)

> * I stand by my opinion that moving files automatically is a bad idea,
> and the FILES issue mentioned in the other thread further backs my point
> (besides that issue, I've mentioned relative symlinks and relative paths
> in general as problems that can break packages when their files are moved).

The issue is simply that MANY packages have a fixed way they install things, and
then other actions (do_install) "move" them around to the expected place.  We
follow that behavior with the fs-perms.txt and "move them around".

The only alternative I can think of is to move the fs-perms.txt processing at
the beginning of do_install.  It could then create the symlinks and directories
mentioned.  Then the install occurs and the tools would need to know how to deal
with it "or fail".  (I don't like the or fail..)

The other problem though is this would introduce a number of (potentially) empty
directories to the a given recipe's install which would kick off the QA
warnings.  (We'd need to resolve that issue first.)

> My proposal would be to change the fs-perms.txt handling to only check
> if nothing conflicts with default symlinks, but not to create or move
> anything. This is another point I'd like to get more input on.

The key here, it can be a lot of additional work (based on experience) to have
to go in and change the locations that things are installing components.  Things
that PROPERLY use gnu autoconf, and other variable systems have an advantage
here, but it's not unusual to see references to '/lib' (i.e. the /lib/firmware
case), or even to '/sbin' for the non "/usr/sbin" case.

We really want this path change, and setup to be seemless for the distro
developer.  Set up the values in the distro configuration (path variables and
fs-perms.txt in some cases) and provide necessary .bbappends or distro specific
version of base-files package.  Theoretically that SHOULD be all that is
required to define the 'filesystem'.

--Mark