From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id C3E0BC25B75 for ; Mon, 13 May 2024 09:47:28 +0000 (UTC) Subject: Re: [PATCH] buildhistory: Fix do_package race issues To: openembedded-core@lists.openembedded.org From: pmi183@gmail.com X-Originating-Location: Rinchoa, Lisbon, PT (78.137.195.161) X-Originating-Platform: Windows Chrome 124 User-Agent: GROUPS.IO Web Poster MIME-Version: 1.0 Date: Mon, 13 May 2024 02:47:19 -0700 References: <20211123135919.26315-1-richard.purdie@linuxfoundation.org> In-Reply-To: <20211123135919.26315-1-richard.purdie@linuxfoundation.org> Message-ID: <25272.1715593639334797374@lists.openembedded.org> Content-Type: multipart/alternative; boundary="7CJUKpxGrMH2B0HFnJdI" List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Mon, 13 May 2024 09:47:28 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/199226 --7CJUKpxGrMH2B0HFnJdI Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable Hi Richard, While using buildhistory, i faced an issue with files-in-package.txt missin= g and digging into the logs i found out: find: =E2=80=98/home/user/src/poky-master/build/tmp/work/core2-64-poky-linu= x/base-passwd/3.6.3/packages-split/*=E2=80=99: No such file or directory Calling `buildhistory_list_pkg_files`=C2=A0 from do_package seems to be acc= essing the dir before being ready and fails. Additionally, using `BUILDHISTORY_RESET` along with `BUILDHISTORY_PRESERVE`= looks to fail to preserve files since there is nothing to handle buildhist= ory/old dir and ends up losing all files marked to preserve. Thank you, Pedro ---------------------------------------- meta/classes/buildhistory.bbclass | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/meta/classes/buildhistory.bbclass b/meta/classes/buildhistory.= bbclass index fd53e92402..4962c53cae 100644 --- a/meta/classes/buildhistory.bbclass +++ b/meta/classes/buildhistory.bbclass @@ -98,11 +98,6 @@ python buildhistory_emit_pkghistory() { if not "package" in (d.getVar('BUILDHISTORY_FEATURES') or "").split(): return 0 - =C2=A0 =C2=A0if d.getVar('BB_CURRENTTASK') in ['package', 'package_setsce= ne']: - =C2=A0 =C2=A0 =C2=A0 =C2=A0# Create files-in-.txt files con= taining a list of files of each recipe's package - =C2=A0 =C2=A0 =C2=A0 =C2=A0bb.build.exec_func("buildhistory_list_pkg_file= s", d) - =C2=A0 =C2=A0 =C2=A0 =C2=A0return 0 - if not d.getVar('BB_CURRENTTASK') in ['packagedata', 'packagedata_setscene'= ]: return 0 @@ -110,6 +105,7 @@ python buildhistory_emit_pkghistory() { import json import shlex import errno + =C2=A0 =C2=A0import shutil pkghistdir =3D d.getVar('BUILDHISTORY_DIR_PACKAGE') oldpkghistdir =3D d.getVar('BUILDHISTORY_OLD_DIR_PACKAGE') @@ -223,6 +219,20 @@ python buildhistory_emit_pkghistory() { items.sort() return ' '.join(items) + =C2=A0 =C2=A0def copypreservedoldpkgdatafiles(pkg, preserve): + =C2=A0 =C2=A0 =C2=A0 =C2=A0if os.path.exists(os.path.join(oldpkghistdir, = pkg)): + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0listofobjs =3D os.listdir(os.pat= h.join(oldpkghistdir, pkg)) + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0for obj in listofobjs: + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0if obj not in pres= erve: + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0cont= inue + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0try: + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0bb.u= tils.mkdirhier(os.path.join(pkghistdir, pkg)) + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0shut= il.copyfile(os.path.join(oldpkghistdir, pkg, obj), os.path.join(pkghistdir,= pkg, obj)) + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0except IOError as = e: + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0bb.n= ote("Unable to copy file. %s" % e) + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0except Environment= Error as e: + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0bb.n= ote("Unable to copy file. %s" % e) + pn =3D d.getVar('PN') pe =3D d.getVar('PE') or "0" pv =3D d.getVar('PV') @@ -250,6 +260,11 @@ python buildhistory_emit_pkghistory() { if not os.path.exists(pkghistdir): bb.utils.mkdirhier(pkghistdir) else: + =C2=A0 =C2=A0 =C2=A0 =C2=A0reset =3D d.getVar("BUILDHISTORY_RESET") + =C2=A0 =C2=A0 =C2=A0 =C2=A0if reset: + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0for pkg in packagelist: + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0copypreservedoldpk= gdatafiles(pkg, preserve) + # Remove files for packages that no longer exist for item in os.listdir(pkghistdir): if item not in preserve: @@ -327,6 +342,10 @@ python buildhistory_emit_pkghistory() { write_pkghistory(pkginfo, d) + =C2=A0 =C2=A0# Only executed when running task `packagedata` + =C2=A0 =C2=A0if d.getVar('BB_CURRENTTASK') =3D=3D 'packagedata': + =C2=A0 =C2=A0 =C2=A0 =C2=A0bb.build.exec_func("buildhistory_list_pkg_file= s", d) + oe.qa.exit_if_errors(d) } -- 2.34.1 --7CJUKpxGrMH2B0HFnJdI Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: quoted-printable Hi Richard,

While using buildhistory, i faced an issue with file= s-in-package.txt missing and digging into the logs i found out:

 

find: ‘/home/user/src/poky-master/build/tmp/work/core2-64-po= ky-linux/base-passwd/3.6.3/packages-split/*’: No such file or directo= ry

Calling `buildhistory_list_pkg_files`  from do_package seems to be acc= essing the dir before being ready and fails.

Additionally, using= `BUILDHISTORY_RESET` along with `BUILDHISTORY_PRESERVE` looks to fail to p= reserve files since there is nothing to handle buildhistory/old dir and end= s up losing all files marked to preserve.

Thank you,
Pedro<= br />
----------------------------------------

 meta/c= lasses/buildhistory.bbclass | 29 ++++++++++++++++++++++++-----
 1= file changed, 24 insertions(+), 5 deletions(-)

diff --git a/met= a/classes/buildhistory.bbclass b/meta/classes/buildhistory.bbclass
ind= ex fd53e92402..4962c53cae 100644
--- a/meta/classes/buildhistory.bbcla= ss
+++ b/meta/classes/buildhistory.bbclass
@@ -98,11 +98,6 @@ pyt= hon buildhistory_emit_pkghistory() {
     if not "packa= ge" in (d.getVar('BUILDHISTORY_FEATURES') or "").split():
   = ;      return 0

-    if d.getVar('BB_CU= RRENTTASK') in ['package', 'package_setscene']:
-      =  # Create files-in-<package-name>.txt files containing a list o= f files of each recipe's package
-        bb.build= .exec_func("buildhistory_list_pkg_files", d)
-       &n= bsp;return 0
-
     if not d.getVar('BB_CURRENTTAS= K') in ['packagedata', 'packagedata_setscene']:
      &= nbsp;  return 0

@@ -110,6 +105,7 @@ python buildhistory_emi= t_pkghistory() {
     import json
    &n= bsp;import shlex
     import errno
+    = import shutil

     pkghistdir =3D d.getVar('BUILD= HISTORY_DIR_PACKAGE')
     oldpkghistdir =3D d.getVar('= BUILDHISTORY_OLD_DIR_PACKAGE')
@@ -223,6 +219,20 @@ python buildhistor= y_emit_pkghistory() {
         items.sort()         return ' '.join(items)

+ &= nbsp;  def copypreservedoldpkgdatafiles(pkg, preserve):
+   =      if os.path.exists(os.path.join(oldpkghistdir, pkg)):+            listofobjs =3D os.listdir(os.= path.join(oldpkghistdir, pkg))
+           &n= bsp;for obj in listofobjs:
+            =    if obj not in preserve:
+         &n= bsp;          continue
+      =          try:
+         =            bb.utils.mkdirhier(os.path.join(pk= ghistdir, pkg))
+               &nb= sp;    shutil.copyfile(os.path.join(oldpkghistdir, pkg, obj), os.= path.join(pkghistdir, pkg, obj))
+           =      except IOError as e:
+        =            bb.note("Unable to copy file. %s"= % e)
+                except = EnvironmentError as e:
+             &nb= sp;      bb.note("Unable to copy file. %s" % e)
+
=      pn =3D d.getVar('PN')
     pe =3D d= .getVar('PE') or "0"
     pv =3D d.getVar('PV')
@@= -250,6 +260,11 @@ python buildhistory_emit_pkghistory() {
  &nbs= p;  if not os.path.exists(pkghistdir):
       = ;  bb.utils.mkdirhier(pkghistdir)
     else:
= +        reset =3D d.getVar("BUILDHISTORY_RESET")
= +        if reset:
+         &= nbsp;  for pkg in packagelist:
+         &nbs= p;      copypreservedoldpkgdatafiles(pkg, preserve)
+         # Remove files for packages that no l= onger exist
         for item in os.listdir(p= kghistdir):
             if item no= t in preserve:
@@ -327,6 +342,10 @@ python buildhistory_emit_pkghistor= y() {

         write_pkghistory(pkginfo= , d)

+    # Only executed when running task `packageda= ta`
+    if d.getVar('BB_CURRENTTASK') =3D=3D 'packagedata':=
+        bb.build.exec_func("buildhistory_list_pk= g_files", d)
+
     oe.qa.exit_if_errors(d)
&= nbsp;}

--
2.34.1 --7CJUKpxGrMH2B0HFnJdI--