[PATCH v3, 1/2] lib/oe/sdk: Add get_extra_sdk_info to reuse code in buildhistory

[PATCH v3, 1/2] lib/oe/sdk: Add get_extra_sdk_info to reuse code in buildhistory

[Francisco Pedraza]

This function is going to be used for generating the target and host manifest files packages for eSDK.
[YOCTO #9038]

Signed-off-by: Francisco Pedraza <francisco.j.pedraza.gonzalez@intel.com>
---
 meta/classes/buildhistory.bbclass | 16 ++++------------
 meta/lib/oe/sdk.py                | 16 ++++++++++++++++
 2 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/meta/classes/buildhistory.bbclass b/meta/classes/buildhistory.bbclass
index 109b375..ab04db7 100644
--- a/meta/classes/buildhistory.bbclass
+++ b/meta/classes/buildhistory.bbclass
@@ -554,25 +554,17 @@ END
 python buildhistory_get_extra_sdkinfo() {
     import operator
     import math
+    # importing oe.sdk due package information req.
+    from oe.sdk import get_extra_sdkinfo
 
     if d.getVar('BB_CURRENTTASK') == 'populate_sdk_ext' and \
             "sdk" in (d.getVar('BUILDHISTORY_FEATURES') or "").split():
-        tasksizes = {}
-        filesizes = {}
-        for root, _, files in os.walk(d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache')):
-            for fn in files:
-                if fn.endswith('.tgz'):
-                    fsize = int(math.ceil(float(os.path.getsize(os.path.join(root, fn))) / 1024))
-                    task = fn.rsplit(':', 1)[1].split('_', 1)[1].split('.')[0]
-                    origtotal = tasksizes.get(task, 0)
-                    tasksizes[task] = origtotal + fsize
-                    filesizes[fn] = fsize
         with open(d.expand('${BUILDHISTORY_DIR_SDK}/sstate-package-sizes.txt'), 'w') as f:
-            filesizes_sorted = sorted(filesizes.items(), key=operator.itemgetter(1, 0), reverse=True)
+            filesizes_sorted = sorted(extra_info['filesizes'].items(), key=operator.itemgetter(1, 0), reverse=True)
             for fn, size in filesizes_sorted:
                 f.write('%10d KiB %s\n' % (size, fn))
         with open(d.expand('${BUILDHISTORY_DIR_SDK}/sstate-task-sizes.txt'), 'w') as f:
-            tasksizes_sorted = sorted(tasksizes.items(), key=operator.itemgetter(1, 0), reverse=True)
+             tasksizes_sorted = sorted(extra_info['tasksizes'].items(), key=operator.itemgetter(1, 0), reverse=True)
             for task, size in tasksizes_sorted:
                 f.write('%10d KiB %s\n' % (size, task))
 }
diff --git a/meta/lib/oe/sdk.py b/meta/lib/oe/sdk.py
index deb823b..d101bf0 100644
--- a/meta/lib/oe/sdk.py
+++ b/meta/lib/oe/sdk.py
@@ -372,5 +372,21 @@ def populate_sdk(d, manifest_dir=None):
     os.environ.clear()
     os.environ.update(env_bkp)
 
+def get_extra_sdk_info(sstate_dir):
+    import operator
+    import math
+
+    extra_info = {}
+    extra_info['tasksizes']
+    for root, _, files in os.walk(sstate_dir):
+        for fn in files:
+            if fn.endswith('.tgz'):
+                fsize = int(math.ceil(float(os.path.getsize(os.path.join(root, fn))) / 1024))
+                task = fn.rsplit(':',1)[1].split('_',1)[1].split(',')[0]
+                origtotal = extra_info['tasksizes'].get(task, 0)
+                extra_info['tasksizes'][task] = origtotal + fsize
+                extra_info['filesizes'][fn] = fsize
+    return extra_info 
+
 if __name__ == "__main__":
     pass
-- 
1.8.3.1

[PATCH v3, 2/2] meta/classes/populate_sdk: Adds support for generating eSDK manifest files

[Francisco Pedraza]

The functionalities to generate SDK and eSDK manifest files are different,
the SDK comes from package information and the eSDK comes from sstate artifacts.
Only execute write_sdk_{host, target}_manifest when is on populate_sdk class.

Adds new functions write_sdk{host, target}_ext_manifest to execute on postprocess
in populate_sdk_ext because at the end we have all the sstate artifacts to generate the manifest.
[YOCTO #9038]

Signed-off-by: Francisco Pedraza <francisco.j.pedraza.gonzalez@intel.com>
---
 meta/classes/populate_sdk_base.bbclass |  6 +++---
 meta/classes/populate_sdk_ext.bbclass  | 27 +++++++++++++++++++++++++++
 2 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/meta/classes/populate_sdk_base.bbclass b/meta/classes/populate_sdk_base.bbclass
index 563582e..6928d02 100644
--- a/meta/classes/populate_sdk_base.bbclass
+++ b/meta/classes/populate_sdk_base.bbclass
@@ -87,9 +87,9 @@ python write_host_sdk_manifest () {
     with open(d.getVar('SDK_HOST_MANIFEST'), 'w') as output:
         output.write(format_pkg_list(pkgs, 'ver'))
 }
-
-POPULATE_SDK_POST_TARGET_COMMAND_append = " write_target_sdk_manifest ; write_sdk_test_data ; "
-POPULATE_SDK_POST_HOST_COMMAND_append = " write_host_sdk_manifest; "
+POPULATE_SDK_POST_TARGET_COMMAND_append = " write_sdk_test_data ; "
+POPULATE_SDK_POST_TARGET_COMMAND_append_task-populate-sdk  = " write_target_sdk_manifest ; "
+POPULATE_SDK_POST_HOST_COMMAND_append_task-populate-sdk = " write_host_sdk_manifest; "
 SDK_PACKAGING_COMMAND = "${@'${SDK_PACKAGING_FUNC};' if '${SDK_PACKAGING_FUNC}' else ''}"
 SDK_POSTPROCESS_COMMAND = " create_sdk_files; check_sdk_sysroots; tar_sdk; ${SDK_PACKAGING_COMMAND} "
 
diff --git a/meta/classes/populate_sdk_ext.bbclass b/meta/classes/populate_sdk_ext.bbclass
index 3bccb14..81d0050 100644
--- a/meta/classes/populate_sdk_ext.bbclass
+++ b/meta/classes/populate_sdk_ext.bbclass
@@ -83,6 +83,33 @@ TOOLCHAIN_OUTPUTNAME_task-populate-sdk-ext = "${TOOLCHAINEXT_OUTPUTNAME}"
 SDK_EXT_TARGET_MANIFEST = "${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.target.manifest"
 SDK_EXT_HOST_MANIFEST = "${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.host.manifest"
 
+python write_target_sdk_ext_manifest () {
+    from oe.sdk import get_extra_sdkinfo
+    sstate_dir = d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache')
+    extra_info = get_extra_sdkinfo(sstate_dir)
+    target = d.getVar('TARGET_SYS')
+    target_multimach = d.getVar('MULTIMACH_TARGET_SYS')
+    with open(d.getVar('SDK_EXT_TARGET_MANIFEST'), 'w') as f:
+        for fn in extra_info['filesizes']:
+            info = fn.split(':') #added
+            if info[2] in (target, target_multimach):
+                f.write("%s %s %s\n" % (info[1], info[2], info[3]))
+}
+
+python write_host_sdk_ext_manifest () {
+    from oe.sdk import get_extra_sdkinfo
+    sstate_dir = d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache')
+    extra_info = get_extra_sdkinfo(sstate_dir)
+    host = d.getVar('BUILD_SYS')
+    with open(d.getVar('SDK_EXT_HOST_MANIFEST'), 'w') as f:
+        for fn in extra_info['filesizes']:
+            info = fn.split(':') #added
+            if info[2] == host:
+                f.write("%s %s %s\n" % (info[1], info[2], info[3]))
+}
+
+SDK_POSTPROCESS_COMMAND_append_task-populate-sdk-ext = "write_target_sdk_ext_manifest; write_host_sdk_ext_manifest; "
+
 SDK_TITLE_task-populate-sdk-ext = "${@d.getVar('DISTRO_NAME') or d.getVar('DISTRO')} Extensible SDK"
 
 def clean_esdk_builddir(d, sdkbasepath):
-- 
1.8.3.1

Re: [PATCH v3, 1/2] lib/oe/sdk: Add get_extra_sdk_info to reuse code in buildhistory

[Joshua Lock]

On Tue, 2017-03-28 at 00:15 -0700, Francisco Pedraza wrote:
> This function is going to be used for generating the target and host
> manifest files packages for eSDK.
> [YOCTO #9038]
> 
> Signed-off-by: Francisco Pedraza <francisco.j.pedraza.gonzalez@intel.
> com>
> ---
>  meta/classes/buildhistory.bbclass | 16 ++++------------
>  meta/lib/oe/sdk.py                | 16 ++++++++++++++++
>  2 files changed, 20 insertions(+), 12 deletions(-)
> 
> diff --git a/meta/classes/buildhistory.bbclass
> b/meta/classes/buildhistory.bbclass
> index 109b375..ab04db7 100644
> --- a/meta/classes/buildhistory.bbclass
> +++ b/meta/classes/buildhistory.bbclass
> @@ -554,25 +554,17 @@ END
>  python buildhistory_get_extra_sdkinfo() {
>      import operator
>      import math

I don't think we need to import math here anymore?

> +    # importing oe.sdk due package information req.

No need for this comment.

> +    from oe.sdk import get_extra_sdkinfo
>  
>      if d.getVar('BB_CURRENTTASK') == 'populate_sdk_ext' and \
>              "sdk" in (d.getVar('BUILDHISTORY_FEATURES') or
> "").split():
> -        tasksizes = {}
> -        filesizes = {}
> -        for root, _, files in
> os.walk(d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache')):
> -            for fn in files:
> -                if fn.endswith('.tgz'):
> -                    fsize =
> int(math.ceil(float(os.path.getsize(os.path.join(root, fn))) / 1024))
> -                    task = fn.rsplit(':', 1)[1].split('_',
> 1)[1].split('.')[0]
> -                    origtotal = tasksizes.get(task, 0)
> -                    tasksizes[task] = origtotal + fsize
> -                    filesizes[fn] = fsize
>          with open(d.expand('${BUILDHISTORY_DIR_SDK}/sstate-package-
> sizes.txt'), 'w') as f:
> -            filesizes_sorted = sorted(filesizes.items(),
> key=operator.itemgetter(1, 0), reverse=True)
> +            filesizes_sorted =
> sorted(extra_info['filesizes'].items(), key=operator.itemgetter(1,
> 0), reverse=True)
>              for fn, size in filesizes_sorted:
>                  f.write('%10d KiB %s\n' % (size, fn))
>          with open(d.expand('${BUILDHISTORY_DIR_SDK}/sstate-task-
> sizes.txt'), 'w') as f:
> -            tasksizes_sorted = sorted(tasksizes.items(),
> key=operator.itemgetter(1, 0), reverse=True)
> +             tasksizes_sorted =
> sorted(extra_info['tasksizes'].items(), key=operator.itemgetter(1,
> 0), reverse=True)

It looks like the indentation changes here?

>              for task, size in tasksizes_sorted:
>                  f.write('%10d KiB %s\n' % (size, task))
>  }
> diff --git a/meta/lib/oe/sdk.py b/meta/lib/oe/sdk.py
> index deb823b..d101bf0 100644
> --- a/meta/lib/oe/sdk.py
> +++ b/meta/lib/oe/sdk.py
> @@ -372,5 +372,21 @@ def populate_sdk(d, manifest_dir=None):
>      os.environ.clear()
>      os.environ.update(env_bkp)
>  
> +def get_extra_sdk_info(sstate_dir):
> +    import operator

I don't think we need operator here?

A docstring for this function would be nice.

> +    import math
> +
> +    extra_info = {}
> +    extra_info['tasksizes']
> +    for root, _, files in os.walk(sstate_dir):
> +        for fn in files:
> +            if fn.endswith('.tgz'):
> +                fsize =
> int(math.ceil(float(os.path.getsize(os.path.join(root, fn))) / 1024))
> +                task =
> fn.rsplit(':',1)[1].split('_',1)[1].split(',')[0]
> +                origtotal = extra_info['tasksizes'].get(task, 0)
> +                extra_info['tasksizes'][task] = origtotal + fsize
> +                extra_info['filesizes'][fn] = fsize
> +    return extra_info 
> +
>  if __name__ == "__main__":
>      pass
> -- 
> 1.8.3.1
>