* [auh][PATCH 01/21] upgradehelper.py: fix checking for do_checkpkg
2017-12-21 16:27 [auh][PATCH 00/21] Further enhancements to automatic upgrade helper Alexander Kanavin
@ 2017-12-21 16:27 ` Alexander Kanavin
2017-12-21 16:27 ` [auh][PATCH 02/21] upgradehelper.py: support upgrade multiple recipes Alexander Kanavin
` (19 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Alexander Kanavin @ 2017-12-21 16:27 UTC (permalink / raw)
To: yocto
From: Robert Yang <liezhi.yang@windriver.com>
The error message in the log is:
Initialising tasks...ERROR: Task do_checkpkg does not exist for target strace [snip]
So line.find("ERROR: Task do_checkpkg does not exist") == 0 doesn't
work, use != -1 to fix the problem.
Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
---
upgradehelper.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/upgradehelper.py b/upgradehelper.py
index 71ee0b0..4797fd8 100755
--- a/upgradehelper.py
+++ b/upgradehelper.py
@@ -646,7 +646,7 @@ class UniverseUpdater(Updater):
self.bb.checkpkg(recipe)
except Error as e:
for line in e.stdout.split('\n'):
- if line.find("ERROR: Task do_checkpkg does not exist") == 0:
+ if line.find("ERROR: Task do_checkpkg does not exist") != -1:
C(" \"distrodata.bbclass\" not inherited. Consider adding "
"the following to your local.conf:\n\n"
"INHERIT =+ \"distrodata\"\n")
--
2.15.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [auh][PATCH 02/21] upgradehelper.py: support upgrade multiple recipes
2017-12-21 16:27 [auh][PATCH 00/21] Further enhancements to automatic upgrade helper Alexander Kanavin
2017-12-21 16:27 ` [auh][PATCH 01/21] upgradehelper.py: fix checking for do_checkpkg Alexander Kanavin
@ 2017-12-21 16:27 ` Alexander Kanavin
2017-12-21 16:27 ` [auh][PATCH 03/21] upgradehelper.py: use UniverseUpdater for all cases Alexander Kanavin
` (18 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Alexander Kanavin @ 2017-12-21 16:27 UTC (permalink / raw)
To: yocto
From: Robert Yang <liezhi.yang@windriver.com>
Now we can use:
$ upgradehelper.py recipe1 recipe2
Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
---
upgradehelper.py | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/upgradehelper.py b/upgradehelper.py
index 4797fd8..c81e748 100755
--- a/upgradehelper.py
+++ b/upgradehelper.py
@@ -75,7 +75,7 @@ def parse_cmdline():
parser = argparse.ArgumentParser(description='Package Upgrade Helper',
formatter_class=argparse.RawTextHelpFormatter,
epilog=help_text)
- parser.add_argument("recipe", help="recipe to be upgraded")
+ parser.add_argument("recipe", nargs = '+', action='store', default='', help="recipe to be upgraded")
parser.add_argument("-t", "--to_version",
help="version to upgrade the recipe to")
@@ -792,7 +792,7 @@ if __name__ == "__main__":
level=debug_levels[args.debug_level - 1])
settings, maintainer_override = parse_config_file(args.config_file)
- recipes = args.recipe.split()
+ recipes = args.recipe
if len(recipes) == 1 and recipes[0] == "all":
updater = UniverseUpdater()
@@ -806,9 +806,12 @@ if __name__ == "__main__":
args.maintainer = "Upgrade Helper <%s>" % \
settings.get('from', 'uh@not.set')
- pkg_list = [(args.recipe, args.to_version, args.maintainer)]
+ pkg_list = [(recipes[0], args.to_version, args.maintainer)]
updater = Updater(args.auto_mode, args.send_emails, args.skip_compilation)
updater.run(pkg_list)
+ elif len(recipes) > 1 and args.to_version:
+ E(" -t is only supported when upgrade one recipe\n")
+ exit(1)
else:
updater = UniverseUpdater(recipes)
updater.run()
--
2.15.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [auh][PATCH 03/21] upgradehelper.py: use UniverseUpdater for all cases
2017-12-21 16:27 [auh][PATCH 00/21] Further enhancements to automatic upgrade helper Alexander Kanavin
2017-12-21 16:27 ` [auh][PATCH 01/21] upgradehelper.py: fix checking for do_checkpkg Alexander Kanavin
2017-12-21 16:27 ` [auh][PATCH 02/21] upgradehelper.py: support upgrade multiple recipes Alexander Kanavin
@ 2017-12-21 16:27 ` Alexander Kanavin
2017-12-21 16:27 ` [auh][PATCH 04/21] modules/steps.py: fix warn when skip compilation Alexander Kanavin
` (17 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Alexander Kanavin @ 2017-12-21 16:27 UTC (permalink / raw)
To: yocto
From: Robert Yang <liezhi.yang@windriver.com>
* Use UniverseUpdater() for the following 3 cases:
+ Upgrade all recipes
+ Upgrade 1 recipe
- '--maintainer' is not a must when any more when use --send-emails, the
maintainer be got from distrodata.
+ Upgrade multiple recipes
* Use "args" as the parameter of UniverseUpdater() and Updater(), this can make
the parameters simple, and easy for extending.
* Make upgrade "all" recipes respect the args, it didn't care --send-emails or
--maintainer which would suprise the user.
Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
---
upgradehelper.py | 68 +++++++++++++++++++++++++-------------------------------
1 file changed, 30 insertions(+), 38 deletions(-)
diff --git a/upgradehelper.py b/upgradehelper.py
index c81e748..e884597 100755
--- a/upgradehelper.py
+++ b/upgradehelper.py
@@ -129,11 +129,12 @@ def parse_config_file(config_file):
return (settings, maintainer_override)
class Updater(object):
- def __init__(self, auto_mode=False, send_email=False, skip_compilation=False):
+ def __init__(self, args):
build_dir = get_build_dir()
self.bb = Bitbake(build_dir)
self.devtool = Devtool()
+ self.args = args
try:
self.base_env = self.bb.env()
@@ -143,7 +144,7 @@ class Updater(object):
E( " Bitbake output:\n%s" % (e.stdout))
exit(1)
- self._set_options(auto_mode, send_email, skip_compilation)
+ self._set_options()
self._make_dirs(build_dir)
@@ -152,7 +153,7 @@ class Updater(object):
self.email_handler = Email(settings)
self.statistics = Statistics()
- def _set_options(self, auto_mode, send_email, skip_compilation):
+ def _set_options(self):
self.opts = {}
self.opts['layer_mode'] = settings.get('layer_mode', '')
if self.opts['layer_mode'] == 'yes':
@@ -176,11 +177,11 @@ class Updater(object):
self.opts['machines'] = settings.get('machines',
'qemux86 qemux86-64 qemuarm qemumips qemuppc').split()
- self.opts['interactive'] = not auto_mode
- self.opts['send_email'] = send_email
+ self.opts['interactive'] = not self.args.auto_mode
+ self.opts['send_email'] = self.args.send_emails
self.opts['author'] = "Upgrade Helper <%s>" % \
settings.get('from', 'uh@not.set')
- self.opts['skip_compilation'] = skip_compilation
+ self.opts['skip_compilation'] = self.args.skip_compilation
self.opts['buildhistory'] = self._buildhistory_is_enabled()
self.opts['testimage'] = self._testimage_is_enabled()
@@ -576,17 +577,25 @@ class Updater(object):
self.send_status_mail(statistics_summary)
class UniverseUpdater(Updater):
- def __init__(self, recipes=None):
- Updater.__init__(self, True, True)
+ def __init__(self, args):
+ Updater.__init__(self, args)
+
+ if len(args.recipe) == 1 and args.recipe[0] == "all":
+ self.recipes = []
+ else:
+ self.recipes = args.recipe
# to filter recipes in upgrade
- if not recipes and self.opts['layer_mode'] == 'yes':
+ if not self.recipes and self.opts['layer_mode'] == 'yes':
# when layer mode is enabled and no recipes are specified
# we need to figure out what recipes are provided by the
# layer to try upgrade
self.recipes = self._get_recipes_by_layer()
- else:
- self.recipes = recipes
+
+ if args.to_version:
+ if len(self.recipes) != 1:
+ E(" -t is only supported when upgrade one recipe\n")
+ exit(1)
# read history file
self.history_file = os.path.join(get_build_dir(), "upgrade-helper", "history.uh")
@@ -665,10 +674,16 @@ class UniverseUpdater(Updater):
pn = row[0]
cur_ver = row[1]
- next_ver = row[2]
+ if self.args.to_version:
+ next_ver = self.args.to_version
+ else:
+ next_ver = row[2]
status = row[11]
revision = row[12]
- maintainer = row[14]
+ if self.args.maintainer:
+ maintainer = self.args.maintainer
+ else:
+ maintainer = row[14]
no_upgrade_reason = row[15]
if status == 'UPDATE' and not no_upgrade_reason:
@@ -792,28 +807,5 @@ if __name__ == "__main__":
level=debug_levels[args.debug_level - 1])
settings, maintainer_override = parse_config_file(args.config_file)
- recipes = args.recipe
-
- if len(recipes) == 1 and recipes[0] == "all":
- updater = UniverseUpdater()
- updater.run()
- elif len(recipes) == 1 and args.to_version:
- if not args.maintainer and args.send_emails:
- E(" For upgrade one recipe and send email you must specify --maintainer\n")
- exit(1)
-
- if not args.maintainer:
- args.maintainer = "Upgrade Helper <%s>" % \
- settings.get('from', 'uh@not.set')
-
- pkg_list = [(recipes[0], args.to_version, args.maintainer)]
- updater = Updater(args.auto_mode, args.send_emails, args.skip_compilation)
- updater.run(pkg_list)
- elif len(recipes) > 1 and args.to_version:
- E(" -t is only supported when upgrade one recipe\n")
- exit(1)
- else:
- updater = UniverseUpdater(recipes)
- updater.run()
-
-
+ updater = UniverseUpdater(args)
+ updater.run()
--
2.15.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [auh][PATCH 04/21] modules/steps.py: fix warn when skip compilation
2017-12-21 16:27 [auh][PATCH 00/21] Further enhancements to automatic upgrade helper Alexander Kanavin
` (2 preceding siblings ...)
2017-12-21 16:27 ` [auh][PATCH 03/21] upgradehelper.py: use UniverseUpdater for all cases Alexander Kanavin
@ 2017-12-21 16:27 ` Alexander Kanavin
2017-12-21 16:27 ` [auh][PATCH 05/21] upgradehelper.py: only check email settings when -e is specified Alexander Kanavin
` (16 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Alexander Kanavin @ 2017-12-21 16:27 UTC (permalink / raw)
To: yocto
From: Robert Yang <liezhi.yang@windriver.com>
Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
---
modules/steps.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/modules/steps.py b/modules/steps.py
index 63f43c8..5bbe38e 100644
--- a/modules/steps.py
+++ b/modules/steps.py
@@ -113,7 +113,7 @@ def _compile(bb, pkg, machine, workdir):
def compile(devtool, bb, git, opts, pkg_ctx):
if opts['skip_compilation']:
- W(" %s: Compilation was skipped by user choice!")
+ W(" %s: Compilation was skipped by user choice!" % pkg_ctx['PN'])
return
for machine in opts['machines']:
--
2.15.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [auh][PATCH 05/21] upgradehelper.py: only check email settings when -e is specified
2017-12-21 16:27 [auh][PATCH 00/21] Further enhancements to automatic upgrade helper Alexander Kanavin
` (3 preceding siblings ...)
2017-12-21 16:27 ` [auh][PATCH 04/21] modules/steps.py: fix warn when skip compilation Alexander Kanavin
@ 2017-12-21 16:27 ` Alexander Kanavin
2017-12-21 16:27 ` [auh][PATCH 06/21] upgradehelper.py: always do upgrade when recipes are specified Alexander Kanavin
` (15 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Alexander Kanavin @ 2017-12-21 16:27 UTC (permalink / raw)
To: yocto
From: Robert Yang <liezhi.yang@windriver.com>
Fixed:
$ upgradehelper.py less
ERROR: smtp host not set! Sending emails disabled!
ERROR: 'From' address not set! Sending emails disabled!
Only check email settings when "-e" is specified can fix the problem.
Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
---
upgradehelper.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/upgradehelper.py b/upgradehelper.py
index e884597..2061bfb 100755
--- a/upgradehelper.py
+++ b/upgradehelper.py
@@ -150,7 +150,8 @@ class Updater(object):
self._add_file_logger()
- self.email_handler = Email(settings)
+ if self.args.send_emails:
+ self.email_handler = Email(settings)
self.statistics = Statistics()
def _set_options(self):
--
2.15.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [auh][PATCH 06/21] upgradehelper.py: always do upgrade when recipes are specified
2017-12-21 16:27 [auh][PATCH 00/21] Further enhancements to automatic upgrade helper Alexander Kanavin
` (4 preceding siblings ...)
2017-12-21 16:27 ` [auh][PATCH 05/21] upgradehelper.py: only check email settings when -e is specified Alexander Kanavin
@ 2017-12-21 16:27 ` Alexander Kanavin
2017-12-21 16:27 ` [auh][PATCH 07/21] upgradehelper.py: print info when recipe is skipped to upgrade Alexander Kanavin
` (14 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Alexander Kanavin @ 2017-12-21 16:27 UTC (permalink / raw)
To: yocto
From: Robert Yang <liezhi.yang@windriver.com>
For example, first run:
$ upgradehelper.py less -d 5
It did the upgrade
Second run:
$ upgradehelper.py less -d 5
DEBUG: Skipping upgrade of less: is in history and not 30 days passed
Let it always do the upgrade makes it easier to use when do upgrade locally.
It will still do the check when the recipe is all.
Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
---
upgradehelper.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/upgradehelper.py b/upgradehelper.py
index 2061bfb..3c40918 100755
--- a/upgradehelper.py
+++ b/upgradehelper.py
@@ -756,7 +756,10 @@ class UniverseUpdater(Updater):
pkgs_list = []
for pkg in self._parse_checkpkg_file(last_checkpkg_file):
- if self._pkg_upgradable(pkg[0], pkg[1], pkg[2]):
+ # Always do the upgrade if recipes are specified
+ if self.recipes and pkg[0] in self.recipes:
+ pkgs_list.append(pkg)
+ elif self._pkg_upgradable(pkg[0], pkg[1], pkg[2]):
pkgs_list.append(pkg)
return pkgs_list
--
2.15.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [auh][PATCH 07/21] upgradehelper.py: print info when recipe is skipped to upgrade
2017-12-21 16:27 [auh][PATCH 00/21] Further enhancements to automatic upgrade helper Alexander Kanavin
` (5 preceding siblings ...)
2017-12-21 16:27 ` [auh][PATCH 06/21] upgradehelper.py: always do upgrade when recipes are specified Alexander Kanavin
@ 2017-12-21 16:27 ` Alexander Kanavin
2017-12-21 16:27 ` [auh][PATCH 08/21] upgradehelper.py: don't build gcc-runtime when --skip-compilation Alexander Kanavin
` (13 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Alexander Kanavin @ 2017-12-21 16:27 UTC (permalink / raw)
To: yocto
From: Robert Yang <liezhi.yang@windriver.com>
This makes debug easier.
Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
---
upgradehelper.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/upgradehelper.py b/upgradehelper.py
index 3c40918..80ea4f4 100755
--- a/upgradehelper.py
+++ b/upgradehelper.py
@@ -691,11 +691,11 @@ class UniverseUpdater(Updater):
pkgs_list.append((pn, cur_ver, next_ver, maintainer, revision))
else:
if no_upgrade_reason:
- D(" Skip package %s (status = %s, current version = %s," \
+ I(" Skip package %s (status = %s, current version = %s," \
" next version = %s, no upgrade reason = %s)" %
(pn, status, cur_ver, next_ver, no_upgrade_reason))
else:
- D(" Skip package %s (status = %s, current version = %s," \
+ I(" Skip package %s (status = %s, current version = %s," \
" next version = %s)" %
(pn, status, cur_ver, next_ver))
return pkgs_list
--
2.15.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [auh][PATCH 08/21] upgradehelper.py: don't build gcc-runtime when --skip-compilation
2017-12-21 16:27 [auh][PATCH 00/21] Further enhancements to automatic upgrade helper Alexander Kanavin
` (6 preceding siblings ...)
2017-12-21 16:27 ` [auh][PATCH 07/21] upgradehelper.py: print info when recipe is skipped to upgrade Alexander Kanavin
@ 2017-12-21 16:27 ` Alexander Kanavin
2017-12-21 16:27 ` [auh][PATCH 09/21] upgradehelper.py: drop automatic mode Alexander Kanavin
` (12 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Alexander Kanavin @ 2017-12-21 16:27 UTC (permalink / raw)
To: yocto
From: Robert Yang <liezhi.yang@windriver.com>
It doesn't make any sense to build it when skip compile.
Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
---
upgradehelper.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/upgradehelper.py b/upgradehelper.py
index 80ea4f4..711d3b3 100755
--- a/upgradehelper.py
+++ b/upgradehelper.py
@@ -468,7 +468,7 @@ class Updater(object):
pkgs_ctx[p]['base_dir'] = self.uh_recipes_all_dir
I(" ############################################################")
- if pkgs_to_upgrade:
+ if pkgs_to_upgrade and not self.args.skip_compilation:
I(" Building gcc runtimes ...")
for machine in self.opts['machines']:
I(" building gcc runtime for %s" % machine)
--
2.15.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [auh][PATCH 09/21] upgradehelper.py: drop automatic mode
2017-12-21 16:27 [auh][PATCH 00/21] Further enhancements to automatic upgrade helper Alexander Kanavin
` (7 preceding siblings ...)
2017-12-21 16:27 ` [auh][PATCH 08/21] upgradehelper.py: don't build gcc-runtime when --skip-compilation Alexander Kanavin
@ 2017-12-21 16:27 ` Alexander Kanavin
2017-12-21 16:27 ` [auh][PATCH 10/21] upgradehelper.py: replace the confusing 'maintainer' command line option Alexander Kanavin
` (11 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Alexander Kanavin @ 2017-12-21 16:27 UTC (permalink / raw)
To: yocto
No longer used since migration to devtool: all upgrades are automatic
Signed-off-by: Alexander Kanavin <alexander.kanavin@linux.intel.com>
---
README | 6 ++----
upgradehelper.py | 6 +-----
2 files changed, 3 insertions(+), 9 deletions(-)
diff --git a/README b/README
index 60dbef9..00a390a 100644
--- a/README
+++ b/README
@@ -134,12 +134,10 @@ Usage
Some manual usage examples:
-* To upgrade the xmodmap recipe to the latest available version,
- interactively:
+* To upgrade the xmodmap recipe to the latest available version:
$ upgrade-helper.py xmodmap
-* To upgrade the xmodmap recipe to a user specified version,
- interactively:
+* To upgrade the xmodmap recipe to a user specified version:
$ upgrade-helper.py xmodmap -t 1.2.3
* To attempt to upgrade all recipes and automatically send email
diff --git a/upgradehelper.py b/upgradehelper.py
index 711d3b3..d5a7112 100755
--- a/upgradehelper.py
+++ b/upgradehelper.py
@@ -60,7 +60,7 @@ from steps import upgrade_steps
from testimage import TestImage
help_text = """Usage examples:
-* To upgrade xmodmap recipe to the latest available version, interactively:
+* To upgrade xmodmap recipe to the latest available version:
$ upgrade-helper.py xmodmap
* To attempt to upgrade all recipes and automatically send email messages
@@ -82,9 +82,6 @@ def parse_cmdline():
parser.add_argument("-m", "--maintainer",
help="maintainer of the recipe")
- parser.add_argument("-a", "--auto-mode", action="store_true", default=False,
- help="disable interactive mode")
-
parser.add_argument("-d", "--debug-level", type=int, default=4, choices=range(1, 6),
help="set the debug level: CRITICAL=1, ERROR=2, WARNING=3, INFO=4, DEBUG=5")
parser.add_argument("-e", "--send-emails", action="store_true", default=False,
@@ -178,7 +175,6 @@ class Updater(object):
self.opts['machines'] = settings.get('machines',
'qemux86 qemux86-64 qemuarm qemumips qemuppc').split()
- self.opts['interactive'] = not self.args.auto_mode
self.opts['send_email'] = self.args.send_emails
self.opts['author'] = "Upgrade Helper <%s>" % \
settings.get('from', 'uh@not.set')
--
2.15.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [auh][PATCH 10/21] upgradehelper.py: replace the confusing 'maintainer' command line option
2017-12-21 16:27 [auh][PATCH 00/21] Further enhancements to automatic upgrade helper Alexander Kanavin
` (8 preceding siblings ...)
2017-12-21 16:27 ` [auh][PATCH 09/21] upgradehelper.py: drop automatic mode Alexander Kanavin
@ 2017-12-21 16:27 ` Alexander Kanavin
2017-12-21 16:27 ` [auh][PATCH 11/21] weeklyjob.sh: set up a temporary branch and clean up afterwards Alexander Kanavin
` (10 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Alexander Kanavin @ 2017-12-21 16:27 UTC (permalink / raw)
To: yocto
It wasn't clear what the option does, and why it need to be on the command line,
so the replacement is a 'global_maintainer_override' config file setting which
does the same thing, in a cleaner fashion.
Signed-off-by: Alexander Kanavin <alexander.kanavin@linux.intel.com>
---
README | 7 ++++++-
upgradehelper.py | 9 +++------
2 files changed, 9 insertions(+), 7 deletions(-)
diff --git a/README b/README
index 00a390a..20b2e5d 100644
--- a/README
+++ b/README
@@ -28,6 +28,7 @@ Setup
--------------- snip ---------------
[maintainer_override]
# mails for recipe upgrades will go to john.doe instead of jane.doe, etc
+# see also the global_maintainer_override option
jane.doe@doe.com=john.doe@doe.com
johhny.bravo@bravo.com=john.doe@doe.com
@@ -38,6 +39,10 @@ blacklist=python glibc gcc
# only recipes belonging to maintainers in whitelist will be attempted
maintainers_whitelist=jane.doe@doe.com john.doe@doe.com johhny.bravo@bravo.com
+# email for all recipe upgrades will go to john.doe, except those listed
+# in specific maintainer_override entries above
+global_maintainer_override=john.doe@doe.com
+
# SMTP server
smtp=smtp.my-server.com:25
@@ -84,7 +89,7 @@ INHERIT =+ "distrodata"
WARNING: if you are using the default maintainers.inc file supplied
with Poky (in meta-yocto) and you don't set a
- maintainers_whitelist or maintainer_override in the
+ maintainers_whitelist or (global_)maintainer_override in the
upgrade-helper configuration as above, and you specify "all"
on the command line, the script will automatically send out
emails to the default maintainers. Please be careful not to
diff --git a/upgradehelper.py b/upgradehelper.py
index d5a7112..4623199 100755
--- a/upgradehelper.py
+++ b/upgradehelper.py
@@ -79,8 +79,6 @@ def parse_cmdline():
parser.add_argument("-t", "--to_version",
help="version to upgrade the recipe to")
- parser.add_argument("-m", "--maintainer",
- help="maintainer of the recipe")
parser.add_argument("-d", "--debug-level", type=int, default=4, choices=range(1, 6),
help="set the debug level: CRITICAL=1, ERROR=2, WARNING=3, INFO=4, DEBUG=5")
@@ -325,6 +323,8 @@ class Updater(object):
if pkg_ctx['MAINTAINER'] in maintainer_override:
to_addr = maintainer_override[pkg_ctx['MAINTAINER']]
+ elif 'global_maintainer_override' in settings:
+ to_addr = settings['global_maintainer_override']
else:
to_addr = pkg_ctx['MAINTAINER']
@@ -677,10 +677,7 @@ class UniverseUpdater(Updater):
next_ver = row[2]
status = row[11]
revision = row[12]
- if self.args.maintainer:
- maintainer = self.args.maintainer
- else:
- maintainer = row[14]
+ maintainer = row[14]
no_upgrade_reason = row[15]
if status == 'UPDATE' and not no_upgrade_reason:
--
2.15.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [auh][PATCH 11/21] weeklyjob.sh: set up a temporary branch and clean up afterwards
2017-12-21 16:27 [auh][PATCH 00/21] Further enhancements to automatic upgrade helper Alexander Kanavin
` (9 preceding siblings ...)
2017-12-21 16:27 ` [auh][PATCH 10/21] upgradehelper.py: replace the confusing 'maintainer' command line option Alexander Kanavin
@ 2017-12-21 16:27 ` Alexander Kanavin
2017-12-21 16:27 ` [auh][PATCH 12/21] Add support for compiling against non-default C libraries Alexander Kanavin
` (9 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Alexander Kanavin @ 2017-12-21 16:27 UTC (permalink / raw)
To: yocto
Signed-off-by: Alexander Kanavin <alexander.kanavin@linux.intel.com>
---
weeklyjob.sh | 14 +++++++++++++-
1 file changed, 13 insertions(+), 1 deletion(-)
diff --git a/weeklyjob.sh b/weeklyjob.sh
index e2c4b1a..8dd0a1c 100755
--- a/weeklyjob.sh
+++ b/weeklyjob.sh
@@ -7,11 +7,23 @@
#
# 00 8 * * 6 auh /home/auh/bin/weeklyjob.sh
+# Re-assign these to match your setup!
auh_dir=~/auto-upgrade-helper
poky_dir=~/poky
build_dir=~/build
+sstate_dir=~/sstate-cache
+
+pushd $poky_dir
+
+# Base the upgrades on poky master
+git fetch origin
+git checkout -B tmp-auh-upgrades origin/master
source $poky_dir/oe-init-build-env $build_dir
$auh_dir/upgradehelper.py all
-#/usr/bin/rsync --delete --password-file /home/auh/rsync.passwd --copy-unsafe-links -zaHS /home/auh/work/ auh@downloads.yoctoproject.org::auh/
+# clean up to avoid the disk filling up
+rm -rf $build_dir/tmp/
+find $sstate_dir -atime +10 -delete
+
+popd
--
2.15.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [auh][PATCH 12/21] Add support for compiling against non-default C libraries
2017-12-21 16:27 [auh][PATCH 00/21] Further enhancements to automatic upgrade helper Alexander Kanavin
` (10 preceding siblings ...)
2017-12-21 16:27 ` [auh][PATCH 11/21] weeklyjob.sh: set up a temporary branch and clean up afterwards Alexander Kanavin
@ 2017-12-21 16:27 ` Alexander Kanavin
2017-12-21 16:27 ` [auh][PATCH 13/21] Add protection measures and information against running with existing build directories Alexander Kanavin
` (8 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Alexander Kanavin @ 2017-12-21 16:27 UTC (permalink / raw)
To: yocto
Particularly, musl is a notorious source of failures and so it's
very useful to do at least one build against it by default.
Signed-off-by: Alexander Kanavin <alexander.kanavin@linux.intel.com>
---
README | 4 +++-
modules/utils/bitbake.py | 7 ++++++-
upgradehelper.py | 2 +-
3 files changed, 10 insertions(+), 3 deletions(-)
diff --git a/README b/README
index 20b2e5d..fabde5c 100644
--- a/README
+++ b/README
@@ -65,7 +65,9 @@ clean_sstate=yes
clean_tmp=yes
# machines to test build with
-machines=qemux86 qemux86-64 qemuarm qemumips qemuppc
+# append _libc-name to test with alternative C library implementations
+# e.g. qemux86_musl
+machines=qemux86 qemux86_musl qemux86-64 qemuarm qemumips qemuppc
# optional features
buildhistory=no
diff --git a/modules/utils/bitbake.py b/modules/utils/bitbake.py
index 8b7c5ed..2ae3a1c 100644
--- a/modules/utils/bitbake.py
+++ b/modules/utils/bitbake.py
@@ -121,7 +121,12 @@ class Bitbake(object):
return self._cmd(recipe, "-c cleansstate")
def complete(self, recipe, machine):
- return self._cmd(recipe, env_var="MACHINE=" + machine)
+ if "_" in machine:
+ machine, libc = machine.split("_")
+ env = "MACHINE={} TCLIBC={}".format(machine, libc)
+ else:
+ env = "MACHINE={}".format(machine)
+ return self._cmd(recipe, env_var=env)
def dependency_graph(self, package_list):
return self._cmd(package_list, "-g")
diff --git a/upgradehelper.py b/upgradehelper.py
index 4623199..a33a3bf 100755
--- a/upgradehelper.py
+++ b/upgradehelper.py
@@ -171,7 +171,7 @@ class Updater(object):
self.git = Git(os.path.dirname(os.getenv('PATH', False).split(':')[0]))
self.poky_git = None
self.opts['machines'] = settings.get('machines',
- 'qemux86 qemux86-64 qemuarm qemumips qemuppc').split()
+ 'qemux86 qemux86-64 qemuarm qemumips qemuppc qemux86_musl').split()
self.opts['send_email'] = self.args.send_emails
self.opts['author'] = "Upgrade Helper <%s>" % \
--
2.15.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [auh][PATCH 13/21] Add protection measures and information against running with existing build directories.
2017-12-21 16:27 [auh][PATCH 00/21] Further enhancements to automatic upgrade helper Alexander Kanavin
` (11 preceding siblings ...)
2017-12-21 16:27 ` [auh][PATCH 12/21] Add support for compiling against non-default C libraries Alexander Kanavin
@ 2017-12-21 16:27 ` Alexander Kanavin
2017-12-21 16:27 ` [auh][PATCH 14/21] upgradehelper.py: do not include absolute paths in the generated tarball Alexander Kanavin
` (7 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Alexander Kanavin @ 2017-12-21 16:27 UTC (permalink / raw)
To: yocto
This should help e.g. with the following issue:
https://bugzilla.yoctoproject.org/show_bug.cgi?id=7174
Signed-off-by: Alexander Kanavin <alexander.kanavin@linux.intel.com>
---
README | 12 ++++++++----
upgradehelper.py | 14 ++++++++++++++
weeklyjob.sh | 2 +-
3 files changed, 23 insertions(+), 5 deletions(-)
diff --git a/README b/README
index fabde5c..27b0977 100644
--- a/README
+++ b/README
@@ -17,10 +17,14 @@ Setup
$ git config --global user.name "Some Name"
$ git config --global user.email "somename@somedomain.com"
-1. Ensure you have a working copy of the build system that is configured
- and ready to run, i.e. you've run the oe-init-build-env script to
- create a build directory and edited local.conf / bblayers.conf as
- desired.
+1. Run the oe-init-build-env script to create a fresh build directory, solely
+ for the purpose of running AUH:
+
+ $ . ./oe-init-build-env build-auh
+
+ Re-using existing build directories and configurations is not recommended
+ as they may contains tweaks that will break AUH or make it behave in
+ undesirable ways.
2. Prepare a configuration file, by default in $BUILDDIR/upgrade-helper/
upgrade-helper.conf, as below:
diff --git a/upgradehelper.py b/upgradehelper.py
index a33a3bf..9110efc 100755
--- a/upgradehelper.py
+++ b/upgradehelper.py
@@ -788,6 +788,8 @@ if __name__ == "__main__":
if not os.getenv('BUILDDIR', False):
E(" You must source oe-init-build-env before running this script!\n")
+ E(" It is recommended to create a fresh build directory with it:\n")
+ E(" $ . oe-init-build-env build-auh\n")
exit(1)
devnull = open(os.devnull, 'wb')
@@ -796,6 +798,18 @@ if __name__ == "__main__":
E(" Git isn't configured please configure user name and email\n")
exit(1)
+ with open(os.getenv('BUILDDIR')+"/conf/local.conf") as f:
+ import re
+ for line in f.readlines():
+ if re.match(r"^MACHINE\s*=", line):
+ E(" The following line found in local.conf - please use ?= or ?== instead as otherwise AUH will not be able to set the desired target machine\n")
+ E(" {}".format(line))
+ exit(1)
+ if re.match(r"^TCLIBC\s*=", line):
+ E(" The following line found in local.conf - please use ?= or ?== instead as otherwise AUH will not be able to set the desired C library\n")
+ E(" {}".format(line))
+ exit(1)
+
signal.signal(signal.SIGINT, close_child_processes)
debug_levels = [log.CRITICAL, log.ERROR, log.WARNING, log.INFO, log.DEBUG]
diff --git a/weeklyjob.sh b/weeklyjob.sh
index 8dd0a1c..ac50461 100755
--- a/weeklyjob.sh
+++ b/weeklyjob.sh
@@ -10,7 +10,7 @@
# Re-assign these to match your setup!
auh_dir=~/auto-upgrade-helper
poky_dir=~/poky
-build_dir=~/build
+build_dir=~/build-tmp-auh-upgrades
sstate_dir=~/sstate-cache
pushd $poky_dir
--
2.15.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [auh][PATCH 14/21] upgradehelper.py: do not include absolute paths in the generated tarball
2017-12-21 16:27 [auh][PATCH 00/21] Further enhancements to automatic upgrade helper Alexander Kanavin
` (12 preceding siblings ...)
2017-12-21 16:27 ` [auh][PATCH 13/21] Add protection measures and information against running with existing build directories Alexander Kanavin
@ 2017-12-21 16:27 ` Alexander Kanavin
2017-12-21 16:27 ` [auh][PATCH 15/21] Do not require configuration options that aren't necessary for running ptests Alexander Kanavin
` (6 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Alexander Kanavin @ 2017-12-21 16:27 UTC (permalink / raw)
To: yocto
Signed-off-by: Alexander Kanavin <alexander.kanavin@linux.intel.com>
---
upgradehelper.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/upgradehelper.py b/upgradehelper.py
index 9110efc..cb48c47 100755
--- a/upgradehelper.py
+++ b/upgradehelper.py
@@ -555,9 +555,11 @@ class Updater(object):
os.path.basename(self.uh_work_dir) + '.tar.gz')
if publish_work_url:
I(" Generating work tarball in %s ..." % work_tarball)
+ tar_cmd = ["tar", "-chzf", work_tarball, "-C", self.uh_base_work_dir, os.path.basename(self.uh_work_dir)]
import subprocess
- if subprocess.call(["tar", "-chzf", work_tarball, self.uh_work_dir]):
+ if subprocess.call(tar_cmd):
E(" Work tarball (%s) generation failed..." % (work_tarball))
+ E(" Tar command: %s" % (" ".join(tar_cmd)))
publish_work_url = ''
statistics_summary = self.statistics.get_summary(
--
2.15.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [auh][PATCH 15/21] Do not require configuration options that aren't necessary for running ptests
2017-12-21 16:27 [auh][PATCH 00/21] Further enhancements to automatic upgrade helper Alexander Kanavin
` (13 preceding siblings ...)
2017-12-21 16:27 ` [auh][PATCH 14/21] upgradehelper.py: do not include absolute paths in the generated tarball Alexander Kanavin
@ 2017-12-21 16:27 ` Alexander Kanavin
2017-12-21 16:27 ` [auh][PATCH 16/21] upgradehelper.py: do not include testimage information in maintainer email messages Alexander Kanavin
` (5 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Alexander Kanavin @ 2017-12-21 16:27 UTC (permalink / raw)
To: yocto
Signed-off-by: Alexander Kanavin <alexander.kanavin@linux.intel.com>
---
README | 6 ++----
upgradehelper.py | 15 ---------------
2 files changed, 2 insertions(+), 19 deletions(-)
diff --git a/README b/README
index 27b0977..acb22ac 100644
--- a/README
+++ b/README
@@ -122,10 +122,8 @@ file:
--------------- snip ---------------
INHERIT += "testimage"
-DISTRO_FEATURES_append = " ptest"
-EXTRA_IMAGE_FEATURES = "debug-tweaks package-management ptest-pkgs"
-# testimage/ptest only work with rpm
-PACKAGE_CLASSES = "package_rpm"
+# Add the following if your distro does not enable by default (poky does)
+#DISTRO_FEATURES_append = " ptest"
--------------- snip ---------------
Also if you are running in a server without X11 session, you need to start
diff --git a/upgradehelper.py b/upgradehelper.py
index cb48c47..5659735 100755
--- a/upgradehelper.py
+++ b/upgradehelper.py
@@ -251,21 +251,6 @@ class Updater(object):
" conf/local.conf.")
exit(1)
- if not "package-management" in self.base_env['EXTRA_IMAGE_FEATURES']:
- E(" testimage requires package-management in EXTRA_IMAGE_FEATURES"\
- " please add to conf/local.conf.")
- exit(1)
-
- if not "ptest-pkgs" in self.base_env['EXTRA_IMAGE_FEATURES']:
- E(" testimage/ptest requires ptest-pkgs in EXTRA_IMAGE_FEATURES"\
- " please add to conf/local.conf.")
- exit(1)
-
- if not "package_rpm" == self.base_env["PACKAGE_CLASSES"]:
- E(" testimage/ptest requires PACKAGE_CLASSES set to package_rpm"\
- " please add to conf/local.conf.")
- exit(1)
-
enabled = True
else:
E(" testimage was enabled in upgrade-helper.conf"\
--
2.15.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [auh][PATCH 16/21] upgradehelper.py: do not include testimage information in maintainer email messages
2017-12-21 16:27 [auh][PATCH 00/21] Further enhancements to automatic upgrade helper Alexander Kanavin
` (14 preceding siblings ...)
2017-12-21 16:27 ` [auh][PATCH 15/21] Do not require configuration options that aren't necessary for running ptests Alexander Kanavin
@ 2017-12-21 16:27 ` Alexander Kanavin
2017-12-21 16:27 ` [auh][PATCH 17/21] testimage.py: clarify what arguments are passed to class constructor Alexander Kanavin
` (4 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Alexander Kanavin @ 2017-12-21 16:27 UTC (permalink / raw)
To: yocto
There's some logic to ascribe failures to specific recipe upgrade commits; it's prone
to mismatches and is lacking context about what else was upgraded. Let's make testimage
support available for local maintainer use only - unattended mass-upgrades will not
try to run testimages.
Signed-off-by: Alexander Kanavin <alexander.kanavin@linux.intel.com>
---
upgradehelper.py | 23 -----------------------
1 file changed, 23 deletions(-)
diff --git a/upgradehelper.py b/upgradehelper.py
index 5659735..1170ca5 100755
--- a/upgradehelper.py
+++ b/upgradehelper.py
@@ -290,17 +290,6 @@ class Updater(object):
"you can fill RECIPE_NO_UPDATE_REASON in respective recipe file so that\n" \
"automatic upgrades would no longer be attempted.\n\n"
- testimage_integration_error = \
- "The recipe *FAILED* in testimage integration. Attached is the log file.\n\n"
-
- testimage_ptest_info = \
- "The recipe has ptest enabled and has been tested with core-image-minimal/ptest \n" \
- "with the next machines %s. Attached is the log file.\n\n"
-
- testimage_info = \
- "The recipe has been tested using %s testimage and succeeded with \n" \
- "the next machines %s. Attached is the log file.\n\n" \
-
mail_footer = \
"Please review the attached files for further information and build/update failures.\n" \
"Any problem please file a bug at https://bugzilla.yoctoproject.org/enter_bug.cgi?product=Automated%20Update%20Handler\n\n" \
@@ -342,18 +331,6 @@ class Updater(object):
if 'patch_file' in pkg_ctx and pkg_ctx['patch_file'] != None:
msg_body += next_steps_info % (os.path.basename(pkg_ctx['patch_file']))
- if self.opts['testimage']:
- if 'integration_error' in pkg_ctx:
- msg_body += testimage_integration_error
- else:
- if 'ptest' in pkg_ctx:
- machines = pkg_ctx['ptest'].keys()
- msg_body += testimage_ptest_info % machines
- if 'testimage' in pkg_ctx:
- machines = pkg_ctx['testimage'].keys()
- msg_body += testimage_info % (settings.get('testimage_name', \
- DEFAULT_TESTIMAGE), machines)
-
msg_body += mail_footer
# Add possible attachments to email
--
2.15.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [auh][PATCH 17/21] testimage.py: clarify what arguments are passed to class constructor
2017-12-21 16:27 [auh][PATCH 00/21] Further enhancements to automatic upgrade helper Alexander Kanavin
` (15 preceding siblings ...)
2017-12-21 16:27 ` [auh][PATCH 16/21] upgradehelper.py: do not include testimage information in maintainer email messages Alexander Kanavin
@ 2017-12-21 16:27 ` Alexander Kanavin
2017-12-21 16:27 ` [auh][PATCH 18/21] testimage.py: run even if no packages were upgraded Alexander Kanavin
` (3 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Alexander Kanavin @ 2017-12-21 16:27 UTC (permalink / raw)
To: yocto
Signed-off-by: Alexander Kanavin <alexander.kanavin@linux.intel.com>
---
modules/testimage.py | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/modules/testimage.py b/modules/testimage.py
index 6c4ed2b..86e817c 100644
--- a/modules/testimage.py
+++ b/modules/testimage.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python
# vim: set ts=4 sw=4 et:
#
# Copyright (c) 2015 Intel Corporation
@@ -42,13 +41,13 @@ def _pn_in_pkgs_ctx(pn, pkgs_ctx):
return None
class TestImage():
- def __init__(self, bb, git, uh_work_dir, opts, *args, **kwargs):
+ def __init__(self, bb, git, uh_work_dir, opts, packages, image):
self.bb = bb
self.git = git
self.uh_work_dir = uh_work_dir
self.opts = opts
- self.pkgs_ctx = args[0]['succeeded'][:]
- self.image = args[1]
+ self.pkgs_ctx = packages['succeeded']
+ self.image = image
os.environ['BB_ENV_EXTRAWHITE'] = os.environ['BB_ENV_EXTRAWHITE'] + \
" TEST_SUITES CORE_IMAGE_EXTRA_INSTALL"
--
2.15.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [auh][PATCH 18/21] testimage.py: run even if no packages were upgraded
2017-12-21 16:27 [auh][PATCH 00/21] Further enhancements to automatic upgrade helper Alexander Kanavin
` (16 preceding siblings ...)
2017-12-21 16:27 ` [auh][PATCH 17/21] testimage.py: clarify what arguments are passed to class constructor Alexander Kanavin
@ 2017-12-21 16:27 ` Alexander Kanavin
2017-12-21 16:27 ` [auh][PATCH 19/21] testimage.py: remove all iterations (over machines, over failing recipes) Alexander Kanavin
` (2 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Alexander Kanavin @ 2017-12-21 16:27 UTC (permalink / raw)
To: yocto
Signed-off-by: Alexander Kanavin <alexander.kanavin@linux.intel.com>
---
modules/testimage.py | 4 ----
1 file changed, 4 deletions(-)
diff --git a/modules/testimage.py b/modules/testimage.py
index 86e817c..1195916 100644
--- a/modules/testimage.py
+++ b/modules/testimage.py
@@ -256,10 +256,6 @@ class TestImage():
return handled
def run(self):
- if len(self.pkgs_ctx) <= 0:
- I(" Testimage was enabled but any upgrade was successful.")
- return
-
I(" Images will test for %s." % ', '.join(self.opts['machines']))
for machine in self.opts['machines']:
I(" Testing images for %s ..." % machine)
--
2.15.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [auh][PATCH 19/21] testimage.py: remove all iterations (over machines, over failing recipes)
2017-12-21 16:27 [auh][PATCH 00/21] Further enhancements to automatic upgrade helper Alexander Kanavin
` (17 preceding siblings ...)
2017-12-21 16:27 ` [auh][PATCH 18/21] testimage.py: run even if no packages were upgraded Alexander Kanavin
@ 2017-12-21 16:27 ` Alexander Kanavin
2017-12-21 16:27 ` [auh][PATCH 20/21] testimage.py: rewrite the logic for building and running testimages Alexander Kanavin
2017-12-21 16:27 ` [auh][PATCH 21/21] testimage.py: collect and write out logs from bitbake and testimage Alexander Kanavin
20 siblings, 0 replies; 22+ messages in thread
From: Alexander Kanavin @ 2017-12-21 16:27 UTC (permalink / raw)
To: yocto
This adds enormously to run time, and is not particularly necessary
when running locally. If something fails, it's best to let the maintainer sort
it out as soon as possible, rather than try to guess what needs to be excluded.
Also, do not build a separate ptest image; ptests will be added
to the standard integration testimage.
Signed-off-by: Alexander Kanavin <alexander.kanavin@linux.intel.com>
---
README | 9 +++--
modules/testimage.py | 107 ++-------------------------------------------------
2 files changed, 9 insertions(+), 107 deletions(-)
diff --git a/README b/README
index acb22ac..84e140f 100644
--- a/README
+++ b/README
@@ -68,9 +68,12 @@ clean_sstate=yes
# clean tmp directory before upgrading
clean_tmp=yes
-# machines to test build with
-# append _libc-name to test with alternative C library implementations
-# e.g. qemux86_musl
+# Machines to test build with.
+# Append _libc-name to test with alternative C library implementations
+# e.g. qemux86_musl.
+#
+# Buildhistory and testimages will be created only for the first
+# machine in the list, as otherwise it adds enormously to AUH run time.
machines=qemux86 qemux86_musl qemux86-64 qemuarm qemumips qemuppc
# optional features
diff --git a/modules/testimage.py b/modules/testimage.py
index 1195916..d2479f5 100644
--- a/modules/testimage.py
+++ b/modules/testimage.py
@@ -139,46 +139,6 @@ class TestImage():
raise IntegrationError(e.stdout, pkg_ctx)
raise e
- def ptest(self, pkgs_ctx, machine):
- image = 'core-image-minimal'
- # should use bitbake API here to trim down the list to only the recipes that inherit ptest
- ptest_pkgs = pkgs_ctx
-
- os.environ['CORE_IMAGE_EXTRA_INSTALL'] = \
- self._get_pkgs_to_install(ptest_pkgs, ptest=True)
- I( " building %s for %s ..." % (image, machine))
- try:
- self.bb.complete(image, machine)
- except Error as e:
- self._handle_image_build_error(image, pkgs_ctx, e)
-
- os.environ['TEST_SUITES'] = "ping ssh _ptest"
- I( " running %s/ptest for %s ..." % (image, machine))
- self.bb.complete("%s -c testimage" % image, machine)
-
- ptest_log_file = self._find_log("ptest.log", machine)
- shutil.copyfile(ptest_log_file,
- os.path.join(self.uh_work_dir, "ptest_%s.log" % machine))
-
- ptest_result = self._parse_ptest_log(ptest_log_file)
- for pn in ptest_result:
- for pkg_ctx in pkgs_ctx:
- if not pn == pkg_ctx['PN']:
- continue
-
- if not 'ptest' in pkg_ctx:
- pkg_ctx['ptest'] = {}
- if not 'ptest_log' in pkg_ctx:
- pkg_ctx['ptest_log'] = os.path.join(pkg_ctx['workdir'],
- "ptest.log")
-
- pkg_ctx['ptest'][machine] = True
- with open(pkg_ctx['ptest_log'], "a+") as f:
- f.write("BEGIN: PTEST for %s\n" % machine)
- for line in ptest_result[pn]:
- f.write(line)
- f.write("END: PTEST for %s\n" % machine)
-
def testimage(self, pkgs_ctx, machine, image):
os.environ['CORE_IMAGE_EXTRA_INSTALL'] = \
self._get_pkgs_to_install(pkgs_ctx)
@@ -213,68 +173,7 @@ class TestImage():
of.write(line)
of.write("END: TESTIMAGE for %s\n" % machine)
- def _log_error(self, e):
- if isinstance(e, Error):
- E(" %s" % e.stdout)
- else:
- import traceback
- tb = traceback.format_exc()
- E("%s" % tb)
-
- def _handle_error(self, e, machine):
- handled = True
-
- if isinstance(e, IntegrationError):
- pkg_ctx = e.pkg_ctx
-
- E(" %s on machine %s failed in integration, removing..."
- % (pkg_ctx['PN'], machine))
-
- with open(os.path.join(pkg_ctx['workdir'],
- 'integration_error.log'), 'a+') as f:
- f.write(e.stdout)
-
- if not pkg_ctx in self.pkgs_ctx:
- E(" Infinite loop IntegrationError trying to " \
- "remove %s twice, see logs.", pkg_ctx['PN'])
- handled = False
- else:
- pkg_ctx['integration_error'] = e
-
- # remove previous build tmp, sstate to avoid QA errors
- # on lower versions
- I(" removing sstate directory ...")
- shutil.rmtree(os.path.join(get_build_dir(), "sstate-cache"))
- I(" removing tmp directory ...")
- shutil.rmtree(os.path.join(get_build_dir(), "tmp"))
-
- self.pkgs_ctx.remove(pkg_ctx)
-
- else:
- handled = False
-
- return handled
-
def run(self):
- I(" Images will test for %s." % ', '.join(self.opts['machines']))
- for machine in self.opts['machines']:
- I(" Testing images for %s ..." % machine)
- while True:
- try:
- self.ptest(self.pkgs_ctx, machine)
- break
- except Exception as e:
- if not self._handle_error(e, machine):
- E(" %s/testimage on machine %s failed" % (self.image, machine))
- self._log_error(e)
- break
-
- while True:
- try:
- self.testimage(self.pkgs_ctx, machine, self.image)
- break
- except Exception as e:
- if not self._handle_error(e, machine):
- E(" %s/testimage on machine %s failed" % (self.image, machine))
- self._log_error(e)
- break
+ machine = self.opts['machines'][0]
+ I(" Testing image for %s ..." % machine)
+ self.testimage(self.pkgs_ctx, machine, self.image)
--
2.15.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [auh][PATCH 20/21] testimage.py: rewrite the logic for building and running testimages
2017-12-21 16:27 [auh][PATCH 00/21] Further enhancements to automatic upgrade helper Alexander Kanavin
` (18 preceding siblings ...)
2017-12-21 16:27 ` [auh][PATCH 19/21] testimage.py: remove all iterations (over machines, over failing recipes) Alexander Kanavin
@ 2017-12-21 16:27 ` Alexander Kanavin
2017-12-21 16:27 ` [auh][PATCH 21/21] testimage.py: collect and write out logs from bitbake and testimage Alexander Kanavin
20 siblings, 0 replies; 22+ messages in thread
From: Alexander Kanavin @ 2017-12-21 16:27 UTC (permalink / raw)
To: yocto
Particularly:
1) determine per-package if ptest is supported, and add to the list
of packages to install accordingly
2) remove broken code that finds image logs in the build directory;
this will be replaced in the following commit
3) remove code that parses said logs for specific package failures;
this is prone to mistmatches and is best left to a human.
Signed-off-by: Alexander Kanavin <alexander.kanavin@linux.intel.com>
---
modules/testimage.py | 115 +++++----------------------------------------------
1 file changed, 10 insertions(+), 105 deletions(-)
diff --git a/modules/testimage.py b/modules/testimage.py
index d2479f5..699d619 100644
--- a/modules/testimage.py
+++ b/modules/testimage.py
@@ -50,129 +50,34 @@ class TestImage():
self.image = image
os.environ['BB_ENV_EXTRAWHITE'] = os.environ['BB_ENV_EXTRAWHITE'] + \
- " TEST_SUITES CORE_IMAGE_EXTRA_INSTALL"
+ " CORE_IMAGE_EXTRA_INSTALL"
- def _get_pkgs_to_install(self, pkgs, ptest=False):
+ def _get_pkgs_to_install(self, pkgs):
pkgs_out = []
- # for provide access to the target
- if ptest:
- pkgs_out.append("dropbear")
- pkgs_out.append("ptest-runner")
-
for c in pkgs:
pkgs_out.append(c['PN'])
- return ' '.join(pkgs_out)
-
- def _parse_ptest_log(self, log_file):
- ptest_results = {}
-
- with open(log_file, "r") as f:
- pn = None
- processing = False
-
- for line in f:
- if not processing:
- m = re.search("^BEGIN: /usr/lib/(.*)/ptest$", line)
- if m:
- pn = m.group(1)
- ptest_results[pn] = []
- processing = True
- else:
- m = re.search("^END: $", line)
- if m:
- pn = None
- processing = False
- else:
- ptest_results[pn].append(line)
-
- return ptest_results
-
- def _find_log(self, name, machine):
- result = []
-
- base_dir = os.path.join(os.getenv('BUILDDIR'), 'tmp', 'work')
- for root, dirs, files in os.walk(base_dir):
- if name in files:
- result.append(os.path.join(root, name))
-
- D("Found logs named %s for machine %s: %s" %(name, machine, result))
- for ptest_log in result:
- if machine in ptest_log:
- D("Picked log: %s" %(ptest_log))
- return ptest_log
-
- def _get_failed_recipe(self, log):
- pn = None
-
- for line in log.splitlines():
- m = re.match("ERROR: QA Issue: ([^ :]*): (.*) not shipped", line)
- if m:
- pn = m.group(1)
- break
-
- m = re.match("ERROR: Logfile of failure stored in: " \
- "(.*/([^/]*)/[^/]*/temp/log\.(.*)\.[0-9]*)", line)
- if m:
- pn = m.group(2)
- break
-
- return pn
-
- def _handle_image_build_error(self, image, pkgs_ctx, e):
- pn = self._get_failed_recipe(e.stdout)
- if pn and pn != image:
- pkg_ctx = _pn_in_pkgs_ctx(pn, pkgs_ctx)
- if pkg_ctx:
- raise IntegrationError(e.stdout, pkg_ctx)
+ I(" Checking if package {} has ptests...".format(c['PN']))
+ if 'PTEST_ENABLED' in self.bb.env(c['PN']):
+ I(" ...yes")
+ pkgs_out.append((c['PN']) + '-ptest')
else:
- pn_env = self.bb.env(pn)
-
- depends = pn_env['DEPENDS'].split()
- rdepends = pn_env['RDEPENDS'].split()
- deps = depends + rdepends
+ I(" ...no")
- for d in deps:
- pkg_ctx = _pn_in_pkgs_ctx(d, pkgs_ctx)
- if pkg_ctx:
- raise IntegrationError(e.stdout, pkg_ctx)
- raise e
+ return ' '.join(pkgs_out)
def testimage(self, pkgs_ctx, machine, image):
os.environ['CORE_IMAGE_EXTRA_INSTALL'] = \
self._get_pkgs_to_install(pkgs_ctx)
-
- if 'TEST_SUITES' in os.environ:
- del os.environ['TEST_SUITES']
+ I( " Installing additional packages to the image: {}".format(os.environ['CORE_IMAGE_EXTRA_INSTALL']))
I( " building %s for %s ..." % (image, machine))
- try:
- self.bb.complete(image, machine)
- except Error as e:
- self._handle_image_build_error(image, pkgs_ctx, e)
+ self.bb.complete(image, machine)
I( " running %s/testimage for %s ..." % (image, machine))
self.bb.complete("%s -c testimage" % image, machine)
- log_file = self._find_log("log.do_testimage", machine)
- shutil.copyfile(log_file,
- os.path.join(self.uh_work_dir, "log_%s.do_testimage" % machine))
- for pkg_ctx in pkgs_ctx:
- if not 'testimage' in pkg_ctx:
- pkg_ctx['testimage'] = {}
- if not 'testimage_log' in pkg_ctx:
- pkg_ctx['testimage_log'] = os.path.join(
- pkg_ctx['workdir'], "log.do_testimage")
-
- pkg_ctx['testimage'][machine] = True
- with open(log_file, "r") as lf:
- with open(pkg_ctx['testimage_log'], "a+") as of:
- of.write("BEGIN: TESTIMAGE for %s\n" % machine)
- for line in lf:
- of.write(line)
- of.write("END: TESTIMAGE for %s\n" % machine)
-
def run(self):
machine = self.opts['machines'][0]
I(" Testing image for %s ..." % machine)
--
2.15.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [auh][PATCH 21/21] testimage.py: collect and write out logs from bitbake and testimage
2017-12-21 16:27 [auh][PATCH 00/21] Further enhancements to automatic upgrade helper Alexander Kanavin
` (19 preceding siblings ...)
2017-12-21 16:27 ` [auh][PATCH 20/21] testimage.py: rewrite the logic for building and running testimages Alexander Kanavin
@ 2017-12-21 16:27 ` Alexander Kanavin
20 siblings, 0 replies; 22+ messages in thread
From: Alexander Kanavin @ 2017-12-21 16:27 UTC (permalink / raw)
To: yocto
Signed-off-by: Alexander Kanavin <alexander.kanavin@linux.intel.com>
---
modules/testimage.py | 31 +++++++++++++++++++++++++++----
1 file changed, 27 insertions(+), 4 deletions(-)
diff --git a/modules/testimage.py b/modules/testimage.py
index 699d619..35e23c9 100644
--- a/modules/testimage.py
+++ b/modules/testimage.py
@@ -49,8 +49,11 @@ class TestImage():
self.pkgs_ctx = packages['succeeded']
self.image = image
+ self.logdir = os.path.join(uh_work_dir, "testimage-logs")
+ os.mkdir(self.logdir)
+
os.environ['BB_ENV_EXTRAWHITE'] = os.environ['BB_ENV_EXTRAWHITE'] + \
- " CORE_IMAGE_EXTRA_INSTALL"
+ " CORE_IMAGE_EXTRA_INSTALL TEST_LOG_DIR TESTIMAGE_UPDATE_VARS"
def _get_pkgs_to_install(self, pkgs):
pkgs_out = []
@@ -70,13 +73,33 @@ class TestImage():
def testimage(self, pkgs_ctx, machine, image):
os.environ['CORE_IMAGE_EXTRA_INSTALL'] = \
self._get_pkgs_to_install(pkgs_ctx)
+ os.environ['TEST_LOG_DIR'] = self.logdir
+ os.environ['TESTIMAGE_UPDATE_VARS'] = 'TEST_LOG_DIR'
I( " Installing additional packages to the image: {}".format(os.environ['CORE_IMAGE_EXTRA_INSTALL']))
I( " building %s for %s ..." % (image, machine))
- self.bb.complete(image, machine)
+ bitbake_create_output = ""
+ bitbake_run_output = ""
+ try:
+ bitbake_create_output = self.bb.complete(image, machine)
+ except Error as e:
+ I( " building the testimage failed! Collecting logs...")
+ bitbake_create_output = e.stdout
+ else:
+ I( " running %s/testimage for %s ..." % (image, machine))
+ try:
+ bitbake_run_output = self.bb.complete("%s -c testimage" % image, machine)
+ except Error as e:
+ I( " running the testimage failed! Collecting logs...")
+ bitbake_run_output = e.stdout
- I( " running %s/testimage for %s ..." % (image, machine))
- self.bb.complete("%s -c testimage" % image, machine)
+ if bitbake_create_output:
+ with open(os.path.join(self.logdir, "bitbake-create-testimage.log"), 'w') as f:
+ f.write(bitbake_create_output)
+ if bitbake_run_output:
+ with open(os.path.join(self.logdir, "bitbake-run-testimage.log"), 'w') as f:
+ f.write(bitbake_run_output)
+ I(" All done! Testimage/ptest/qemu logs are collected to {}".format(self.logdir))
def run(self):
machine = self.opts['machines'][0]
--
2.15.1
^ permalink raw reply related [flat|nested] 22+ messages in thread