* [auh][PATCH 01/20] upgradehelper.py: do not error out if testimage or buildhistory are enabled in bitbake conf, but not in AUH conf
2017-12-14 16:35 [auh][PATCH 00/20] Porting AUH to devtool and making it suitable for local use Alexander Kanavin
@ 2017-12-14 16:35 ` Alexander Kanavin
2017-12-14 16:35 ` [auh][PATCH 02/20] upgradehelper.py: always run checkpkg Alexander Kanavin
` (18 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Alexander Kanavin @ 2017-12-14 16:35 UTC (permalink / raw)
To: yocto
From: Alexander Kanavin <alex.kanavin@gmail.com>
Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
---
upgradehelper.py | 12 ------------
1 file changed, 12 deletions(-)
diff --git a/upgradehelper.py b/upgradehelper.py
index 8e5466e..1356aef 100755
--- a/upgradehelper.py
+++ b/upgradehelper.py
@@ -240,12 +240,6 @@ class Updater(object):
" but isn't INHERIT in conf/local.conf, if you want"\
" to enable please set.")
exit(1)
- else:
- if 'buildhistory' in self.base_env['INHERIT']:
- E(" Buildhistory was INHERIT in conf/local.conf"\
- " but buildhistory=yes isn't in upgrade-helper.conf,"\
- " if you want to enable please set.")
- exit(1)
return enabled
@@ -280,12 +274,6 @@ class Updater(object):
" but isn't INHERIT in conf/local.conf, if you want"\
" to enable please set.")
exit(1)
- else:
- if 'testimage' in self.base_env['INHERIT']:
- E(" testimage was INHERIT in conf/local.conf"\
- " but testimage=yes isn't in upgrade-helper.conf,"\
- " if you want to enable please set.")
- exit(1)
return enabled
--
2.15.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [auh][PATCH 02/20] upgradehelper.py: always run checkpkg
2017-12-14 16:35 [auh][PATCH 00/20] Porting AUH to devtool and making it suitable for local use Alexander Kanavin
2017-12-14 16:35 ` [auh][PATCH 01/20] upgradehelper.py: do not error out if testimage or buildhistory are enabled in bitbake conf, but not in AUH conf Alexander Kanavin
@ 2017-12-14 16:35 ` Alexander Kanavin
2017-12-14 16:35 ` [auh][PATCH 03/20] upgradehelper.py: do not manipulate git branches Alexander Kanavin
` (17 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Alexander Kanavin @ 2017-12-14 16:35 UTC (permalink / raw)
To: yocto
From: Alexander Kanavin <alex.kanavin@gmail.com>
There was code to skip upstream version check if it was performed
recently, which was actually broken as it didn't verify that
checkpkg.csv already contains the package we want to update.
Let's just drop the whole logic and run checkpkg always.
Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
---
upgradehelper.py | 33 ++-------------------------------
1 file changed, 2 insertions(+), 31 deletions(-)
diff --git a/upgradehelper.py b/upgradehelper.py
index 1356aef..651345b 100755
--- a/upgradehelper.py
+++ b/upgradehelper.py
@@ -829,43 +829,14 @@ class UniverseUpdater(Updater):
return True
def _get_packages_to_upgrade(self, packages=None):
- last_date_checked = None
- last_master_commit = None
- last_checkpkg_file = None
- current_date = date.isoformat(date.today())
- try:
- stdout = self.git.last_commit("master")
- cur_master_commit = stdout
- except Error:
- cur_master_commit = "unknown"
-
- if os.path.exists(get_build_dir() + "/upgrade-helper/last_checkpkg_run"):
- with open(get_build_dir() + "/upgrade-helper/last_checkpkg_run") as last_check:
- line = last_check.read()
- last_date_checked = line.split(',')[0]
- last_master_commit = line.split(',')[1]
- last_checkpkg_file = line.split(',')[2]
- if not os.path.exists(last_checkpkg_file):
- last_checkpkg_file = None
-
- if last_master_commit != cur_master_commit or last_date_checked != current_date or \
- last_checkpkg_file is None:
- self._check_upstream_versions()
- last_checkpkg_file = os.path.realpath(get_build_dir() + "/tmp/log/checkpkg.csv")
- else:
- I(" Using last checkpkg.csv file since last master commit and last"
- " check date are the same ...")
+ self._check_upstream_versions()
+ last_checkpkg_file = os.path.realpath(get_build_dir() + "/tmp/log/checkpkg.csv")
pkgs_list = []
for pkg in self._parse_checkpkg_file(last_checkpkg_file):
if self._pkg_upgradable(pkg[0], pkg[1], pkg[2]):
pkgs_list.append(pkg)
- # Update last_checkpkg_run only after the version check has been completed
- with open(get_build_dir() + "/upgrade-helper/last_checkpkg_run", "w+") as last_check:
- last_check.write(current_date + "," + cur_master_commit + "," +
- last_checkpkg_file)
-
return pkgs_list
def _update_history(self, pn, new_ver, maintainer, upgrade_status):
--
2.15.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [auh][PATCH 03/20] upgradehelper.py: do not manipulate git branches.
2017-12-14 16:35 [auh][PATCH 00/20] Porting AUH to devtool and making it suitable for local use Alexander Kanavin
2017-12-14 16:35 ` [auh][PATCH 01/20] upgradehelper.py: do not error out if testimage or buildhistory are enabled in bitbake conf, but not in AUH conf Alexander Kanavin
2017-12-14 16:35 ` [auh][PATCH 02/20] upgradehelper.py: always run checkpkg Alexander Kanavin
@ 2017-12-14 16:35 ` Alexander Kanavin
2017-12-14 16:35 ` [auh][PATCH 04/20] upgradehelper.py: support updates to a new vcs revision Alexander Kanavin
` (16 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Alexander Kanavin @ 2017-12-14 16:35 UTC (permalink / raw)
To: yocto
From: Alexander Kanavin <alex.kanavin@gmail.com>
Let's just write the commits into the current branch, and leave
branch management to some other tool.
Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
---
upgradehelper.py | 25 -------------------------
1 file changed, 25 deletions(-)
diff --git a/upgradehelper.py b/upgradehelper.py
index 651345b..5fb65cd 100755
--- a/upgradehelper.py
+++ b/upgradehelper.py
@@ -695,30 +695,6 @@ class UniverseUpdater(Updater):
return recipes
- def _update_master(self):
- if self.opts['layer_mode'] == 'yes':
- I(" Sync poky master ...")
- self.poky_git.reset_hard()
- self.poky_git.clean_untracked()
- self.poky_git.checkout_branch("master")
- self.poky_git.pull()
-
- I(" Drop all uncommited changes (including untracked) ...")
- self.git.reset_hard()
- self.git.clean_untracked()
-
- self.git.checkout_branch("master")
- try:
- self.git.delete_branch("upgrades")
- except Error:
- pass
- if self.opts['layer_mode'] == 'yes':
- I(" Sync %s master ..." % self.opts['layer_name'])
- else:
- I(" Sync poky master ...")
- self.git.pull()
- self.git.create_branch("upgrades")
-
def _prepare(self):
if settings.get("clean_sstate", "no") == "yes" and \
os.path.exists(os.path.join(get_build_dir(), "sstate-cache")):
@@ -857,7 +833,6 @@ class UniverseUpdater(Updater):
self._get_status_msg(pkg_ctx['error']))
def run(self):
- self._update_master()
self._prepare()
super(UniverseUpdater, self).run()
--
2.15.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [auh][PATCH 04/20] upgradehelper.py: support updates to a new vcs revision.
2017-12-14 16:35 [auh][PATCH 00/20] Porting AUH to devtool and making it suitable for local use Alexander Kanavin
` (2 preceding siblings ...)
2017-12-14 16:35 ` [auh][PATCH 03/20] upgradehelper.py: do not manipulate git branches Alexander Kanavin
@ 2017-12-14 16:35 ` Alexander Kanavin
2017-12-14 16:35 ` [auh][PATCH 05/20] upgradehelper.py: add the old recipe version in pkg_ctx Alexander Kanavin
` (15 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Alexander Kanavin @ 2017-12-14 16:35 UTC (permalink / raw)
To: yocto
From: Alexander Kanavin <alex.kanavin@gmail.com>
Devtool has support for this, and it's useful when the upstream
never issues new versions; in this situation we can simply
attempt to update to the latest revision.
Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
---
upgradehelper.py | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/upgradehelper.py b/upgradehelper.py
index 5fb65cd..c11993a 100755
--- a/upgradehelper.py
+++ b/upgradehelper.py
@@ -526,13 +526,14 @@ class Updater(object):
pkgs_ctx = {}
I(" ########### The list of recipes to be upgraded #############")
- for p, v, m in pkgs_to_upgrade:
- I(" %s, %s, %s" % (p, v, m))
+ for p, v, m, r in pkgs_to_upgrade:
+ I(" %s, %s, %s, %s" % (p, v, m, r))
pkgs_ctx[p] = {}
pkgs_ctx[p]['PN'] = p
pkgs_ctx[p]['NPV'] = v
pkgs_ctx[p]['MAINTAINER'] = m
+ pkgs_ctx[p]['NSRCREV'] = r
pkgs_ctx[p]['base_dir'] = self.uh_recipes_all_dir
I(" ############################################################")
@@ -555,7 +556,7 @@ class Updater(object):
succeeded_pkgs_ctx = []
failed_pkgs_ctx = []
attempted_pkgs = 0
- for pn, _, _ in pkgs_to_upgrade:
+ for pn, _, _, _ in pkgs_to_upgrade:
pkg_ctx = pkgs_ctx[pn]
pkg_ctx['error'] = None
@@ -738,11 +739,12 @@ class UniverseUpdater(Updater):
cur_ver = row[1]
next_ver = row[2]
status = row[11]
+ revision = row[12]
maintainer = row[14]
no_upgrade_reason = row[15]
if status == 'UPDATE' and not no_upgrade_reason:
- pkgs_list.append((pn, next_ver, maintainer))
+ pkgs_list.append((pn, next_ver, maintainer, revision))
else:
if no_upgrade_reason:
D(" Skip package %s (status = %s, current version = %s," \
--
2.15.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [auh][PATCH 05/20] upgradehelper.py: add the old recipe version in pkg_ctx
2017-12-14 16:35 [auh][PATCH 00/20] Porting AUH to devtool and making it suitable for local use Alexander Kanavin
` (3 preceding siblings ...)
2017-12-14 16:35 ` [auh][PATCH 04/20] upgradehelper.py: support updates to a new vcs revision Alexander Kanavin
@ 2017-12-14 16:35 ` Alexander Kanavin
2017-12-14 16:35 ` [auh][PATCH 06/20] Add devtool support for upgrading recipes Alexander Kanavin
` (14 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Alexander Kanavin @ 2017-12-14 16:35 UTC (permalink / raw)
To: yocto
From: Alexander Kanavin <alex.kanavin@gmail.com>
So that it can be later used in the commit message.
Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
---
upgradehelper.py | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/upgradehelper.py b/upgradehelper.py
index c11993a..9a59941 100755
--- a/upgradehelper.py
+++ b/upgradehelper.py
@@ -526,12 +526,13 @@ class Updater(object):
pkgs_ctx = {}
I(" ########### The list of recipes to be upgraded #############")
- for p, v, m, r in pkgs_to_upgrade:
- I(" %s, %s, %s, %s" % (p, v, m, r))
+ for p, ov, nv, m, r in pkgs_to_upgrade:
+ I(" %s, %s, %s, %s, %s" % (p, ov, nv, m, r))
pkgs_ctx[p] = {}
pkgs_ctx[p]['PN'] = p
- pkgs_ctx[p]['NPV'] = v
+ pkgs_ctx[p]['PV'] = ov
+ pkgs_ctx[p]['NPV'] = nv
pkgs_ctx[p]['MAINTAINER'] = m
pkgs_ctx[p]['NSRCREV'] = r
@@ -556,7 +557,7 @@ class Updater(object):
succeeded_pkgs_ctx = []
failed_pkgs_ctx = []
attempted_pkgs = 0
- for pn, _, _, _ in pkgs_to_upgrade:
+ for pn, _, _, _, _ in pkgs_to_upgrade:
pkg_ctx = pkgs_ctx[pn]
pkg_ctx['error'] = None
@@ -744,7 +745,7 @@ class UniverseUpdater(Updater):
no_upgrade_reason = row[15]
if status == 'UPDATE' and not no_upgrade_reason:
- pkgs_list.append((pn, next_ver, maintainer, revision))
+ pkgs_list.append((pn, cur_ver, next_ver, maintainer, revision))
else:
if no_upgrade_reason:
D(" Skip package %s (status = %s, current version = %s," \
--
2.15.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [auh][PATCH 06/20] Add devtool support for upgrading recipes
2017-12-14 16:35 [auh][PATCH 00/20] Porting AUH to devtool and making it suitable for local use Alexander Kanavin
` (4 preceding siblings ...)
2017-12-14 16:35 ` [auh][PATCH 05/20] upgradehelper.py: add the old recipe version in pkg_ctx Alexander Kanavin
@ 2017-12-14 16:35 ` Alexander Kanavin
2017-12-14 16:35 ` [auh][PATCH 07/20] buildhistory.py: replace cleanall with cleansstate Alexander Kanavin
` (13 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Alexander Kanavin @ 2017-12-14 16:35 UTC (permalink / raw)
To: yocto
From: Alexander Kanavin <alex.kanavin@gmail.com>
devtool has a much better support for rebasing patches,
can be used interactively when things go wrong, and we
also need to avoid having two codebases for updating
recipes that broadly do the same thing.
Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
---
modules/errors.py | 7 +-
modules/steps.py | 191 ++++++++++++++++++++++-------------------------
modules/utils/devtool.py | 41 ++++++++++
upgradehelper.py | 30 ++++----
4 files changed, 151 insertions(+), 118 deletions(-)
create mode 100644 modules/utils/devtool.py
diff --git a/modules/errors.py b/modules/errors.py
index ba87180..ad56215 100644
--- a/modules/errors.py
+++ b/modules/errors.py
@@ -33,8 +33,11 @@ class Error(Exception):
class MaintainerError(Error):
""" Class for group error that can be sent to Maintainer's """
- def __init__(self, message=None, stdout=None, stderr=None):
- super(MaintainerError, self).__init__(message, stdout, stderr)
+
+class DevtoolError(Error):
+ """ Class for devtool errors """
+ def __str__(self):
+ return "Failed (devtool error)"
class FetchError(Error):
def __init__(self):
diff --git a/modules/steps.py b/modules/steps.py
index 0c527a4..63f43c8 100644
--- a/modules/steps.py
+++ b/modules/steps.py
@@ -21,6 +21,8 @@
import os
import sys
import subprocess
+import shutil
+import re
from logging import debug as D
from logging import info as I
@@ -31,40 +33,7 @@ from logging import critical as C
from errors import *
from buildhistory import BuildHistory
-from recipe.base import Recipe
-from recipe.git import GitRecipe
-from recipe.svn import SvnRecipe
-
-def clean_repo(bb, git, opts, pkg_ctx):
- git.checkout_branch("master")
-
- try:
- git.delete_branch("remove_patches")
- except:
- pass
- try:
- git.delete_branch("upgrades")
- except:
- pass
-
- git.reset_hard()
- git.create_branch("upgrades")
-
-def load_env(bb, git, opts, pkg_ctx):
- stdout = git.status()
- if stdout != "":
- if opts['interactive']:
- W(" %s: git repository has uncommited work which will be dropped!" \
- " Proceed? (y/N)" % pkg_ctx['PN'])
- answer = sys.stdin.readline().strip().upper()
- if answer == '' or answer != 'Y':
- I(" %s: User abort!" % pkg_ctx['PN'])
- exit(1)
-
- I(" %s: Dropping uncommited work!" % pkg_ctx['PN'])
- git.reset_hard()
- git.clean_untracked()
-
+def load_env(devtool, bb, git, opts, pkg_ctx):
pkg_ctx['env'] = bb.env(pkg_ctx['PN'])
pkg_ctx['workdir'] = os.path.join(pkg_ctx['base_dir'], pkg_ctx['PN'])
os.mkdir(pkg_ctx['workdir'])
@@ -73,97 +42,117 @@ def load_env(bb, git, opts, pkg_ctx):
if pkg_ctx['env']['PV'] == pkg_ctx['NPV']:
raise UpgradeNotNeededError
-def detect_recipe_type(bb, git, opts, pkg_ctx):
- if pkg_ctx['env']['SRC_URI'].find("ftp://") != -1 or \
- pkg_ctx['env']['SRC_URI'].find("http://") != -1 or \
- pkg_ctx['env']['SRC_URI'].find("https://") != -1:
- recipe = Recipe
- elif pkg_ctx['env']['SRC_URI'].find("git://") != -1:
- recipe = GitRecipe
- else:
- raise UnsupportedProtocolError
-
- pkg_ctx['recipe'] = recipe(pkg_ctx['env'], pkg_ctx['NPV'],
- opts['interactive'], pkg_ctx['workdir'],
- pkg_ctx['recipe_dir'], bb, git)
-
-def buildhistory_init(bb, git, opts, pkg_ctx):
+def buildhistory_init(devtool, bb, git, opts, pkg_ctx):
if not opts['buildhistory']:
return
pkg_ctx['buildhistory'] = BuildHistory(bb, pkg_ctx['PN'],
pkg_ctx['workdir'])
I(" %s: Initial buildhistory for %s ..." % (pkg_ctx['PN'],
- opts['machines']))
- pkg_ctx['buildhistory'].init(opts['machines'])
-
-def unpack_original(bb, git, opts, pkg_ctx):
- pkg_ctx['recipe'].unpack()
-
-def pack_original_workdir(bb, git, opts, pkg_ctx):
- recipe_workdir = os.path.dirname(pkg_ctx['env']['S'])
- pkg_ctx['recipe_workdir_tarball'] = os.path.join(pkg_ctx['workdir'],
- 'original_workdir.tar.gz')
-
- try:
- subprocess.call(["tar", "-chzf", pkg_ctx['recipe_workdir_tarball'],
- recipe_workdir], stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
- except:
- W(" %s, Can't compress original workdir, if license diff" \
- " is needed will show full file." % pkg_ctx['PN'])
-
-def rename(bb, git, opts, pkg_ctx):
- pkg_ctx['recipe'].rename()
-
- pkg_ctx['env'] = bb.env(pkg_ctx['PN'])
-
- pkg_ctx['recipe'].update_env(pkg_ctx['env'])
-
-def cleanall(bb, git, opts, pkg_ctx):
- pkg_ctx['recipe'].cleanall()
-
-def fetch(bb, git, opts, pkg_ctx):
- pkg_ctx['recipe'].fetch()
+ opts['machines'][:1]))
+ pkg_ctx['buildhistory'].init(opts['machines'][:1])
+
+def _extract_license_diff(devtool_output):
+ licenseinfo = []
+ for line in devtool_output.split('\n'):
+ if line.startswith("NOTE: New recipe is"):
+ recipepath = line.split()[4]
+ with open(recipepath, 'rb') as f:
+ lines = f.readlines()
+
+ extracting = False
+ with open(recipepath, 'wb') as f:
+ for line in lines:
+ if line.startswith(b'# FIXME: the LIC_FILES_CHKSUM'):
+ extracting = True
+ elif extracting == True and not line.startswith(b'#') and len(line) > 1:
+ extracting = False
+ if extracting == True:
+ licenseinfo.append(line[2:])
+ else:
+ f.write(line)
+ D(" License diff extracted: {}".format(b"".join(licenseinfo).decode('utf-8')))
+ return licenseinfo
+
+def devtool_upgrade(devtool, bb, git, opts, pkg_ctx):
+ if pkg_ctx['NPV'].endswith("new-commits-available"):
+ pkg_ctx['commit_msg'] = "{}: upgrade to latest revision".format(pkg_ctx['PN'])
+ else:
+ pkg_ctx['commit_msg'] = "{}: upgrade {} -> {}".format(pkg_ctx['PN'], pkg_ctx['PV'], pkg_ctx['NPV'])
-def unpack_original_workdir(bb, git, opts, pkg_ctx):
try:
- subprocess.call(["tar", "-xhzf", pkg_ctx['recipe_workdir_tarball'],
- "-C", "/"], stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
- os.unlink(pkg_ctx['recipe_workdir_tarball'])
- except:
- pass
-
-def compile(bb, git, opts, pkg_ctx):
+ devtool_output = devtool.upgrade(pkg_ctx['PN'], pkg_ctx['NPV'], pkg_ctx['NSRCREV'])
+ except DevtoolError as e1:
+ try:
+ devtool_output = devtool.reset(pkg_ctx['PN'])
+ _rm_source_tree(devtool_output)
+ except DevtoolError as e2:
+ pass
+ raise e1
+
+ license_diff_info = _extract_license_diff(devtool_output)
+ if len(license_diff_info) > 0:
+ pkg_ctx['license_diff_fn'] = "license-diff.txt"
+ with open(os.path.join(pkg_ctx['workdir'], pkg_ctx['license_diff_fn']), 'wb') as f:
+ f.write(b"".join(license_diff_info))
+
+ D(" 'devtool upgrade' printed:\n%s" %(devtool_output))
+
+def _compile(bb, pkg, machine, workdir):
+ try:
+ bb.complete(pkg, machine)
+ except Error as e:
+ with open("{}/bitbake-output-{}.txt".format(workdir, machine), 'w') as f:
+ f.write(e.stdout)
+ for line in e.stdout.split("\n"):
+ # version going backwards is not a real error
+ if re.match(".* went backwards which would break package feeds .*", line):
+ break
+ else:
+ raise CompilationError()
+
+def compile(devtool, bb, git, opts, pkg_ctx):
if opts['skip_compilation']:
W(" %s: Compilation was skipped by user choice!")
return
for machine in opts['machines']:
- I(" %s: compiling for %s ..." % (pkg_ctx['PN'], machine))
- pkg_ctx['recipe'].compile(machine)
+ I(" %s: compiling upgraded version for %s ..." % (pkg_ctx['PN'], machine))
+ _compile(bb, pkg_ctx['PN'], machine, pkg_ctx['workdir'])
if opts['buildhistory']:
pkg_ctx['buildhistory'].add()
-def buildhistory_diff(bb, git, opts, pkg_ctx):
+def buildhistory_diff(devtool, bb, git, opts, pkg_ctx):
if not opts['buildhistory']:
return
I(" %s: Checking buildhistory ..." % pkg_ctx['PN'])
pkg_ctx['buildhistory'].diff()
+def _rm_source_tree(devtool_output):
+ for line in devtool_output.split("\n"):
+ if line.startswith("NOTE: Leaving source tree"):
+ srctree = line.split()[4]
+ shutil.rmtree(srctree)
+
+def devtool_finish(devtool, bb, git, opts, pkg_ctx):
+ try:
+ devtool_output = devtool.finish(pkg_ctx['PN'], pkg_ctx['recipe_dir'])
+ _rm_source_tree(devtool_output)
+ D(" 'devtool finish' printed:\n%s" %(devtool_output))
+ except DevtoolError as e1:
+ try:
+ devtool_output = devtool.reset(pkg_ctx['PN'])
+ _rm_source_tree(devtool_output)
+ except DevtoolError as e2:
+ pass
+ raise e1
+
upgrade_steps = [
- (clean_repo, "Cleaning git repository of temporary branch ..."),
(load_env, "Loading environment ..."),
- (detect_recipe_type, None),
(buildhistory_init, None),
- (unpack_original, "Fetch & unpack original version ..."),
- (pack_original_workdir, None),
- (rename, "Renaming recipes, reset PR (if exists) ..."),
- (cleanall, "Clean all ..."),
- (fetch, "Fetch new version (old checksums) ..."),
- (unpack_original_workdir, None),
+ (devtool_upgrade, "Running 'devtool upgrade' ..."),
+ (devtool_finish, "Running 'devtool finish' ..."),
(compile, None),
- (buildhistory_diff, None)
+ (buildhistory_diff, None),
]
diff --git a/modules/utils/devtool.py b/modules/utils/devtool.py
new file mode 100644
index 0000000..7f5b5d4
--- /dev/null
+++ b/modules/utils/devtool.py
@@ -0,0 +1,41 @@
+import os
+import logging as log
+from logging import debug as D
+
+from utils.bitbake import *
+
+class Devtool(object):
+ def __init__(self):
+ super(Devtool, self).__init__()
+
+ def _cmd(self, operation):
+ cmd = "devtool " + operation
+ try:
+ D("Running '%s'" %(cmd))
+ stdout, stderr = bb.process.run(cmd)
+ except bb.process.ExecutionError as e:
+ D("%s returned:\n%s" % (cmd, e.__str__()))
+ raise DevtoolError("The following devtool command failed: " + operation,
+ e.stdout, e.stderr)
+
+ return stdout
+
+ def upgrade(self, recipe, version = None, revision = None):
+ cmd = " upgrade " + recipe
+ if version and not version.endswith("-new-commits-available"):
+ cmd = cmd + " -V " + version
+ if revision and revision != "N/A":
+ cmd = cmd + " -S " + revision
+ return self._cmd(cmd)
+
+ def finish(self, recipe, layer):
+ cmd = " finish -f " + recipe + " " + layer
+ return self._cmd(cmd)
+
+ def reset(self, recipe = None):
+ if recipe:
+ cmd = " reset -n " + recipe
+ else:
+ cmd = " reset -a"
+ return self._cmd(cmd)
+
diff --git a/upgradehelper.py b/upgradehelper.py
index 9a59941..5149993 100755
--- a/upgradehelper.py
+++ b/upgradehelper.py
@@ -51,6 +51,7 @@ sys.path.insert(1, os.path.join(os.path.abspath(
from errors import *
from utils.git import Git
+from utils.devtool import Devtool
from utils.bitbake import *
from utils.emailhandler import Email
@@ -132,6 +133,7 @@ class Updater(object):
build_dir = get_build_dir()
self.bb = Bitbake(build_dir)
+ self.devtool = Devtool()
try:
self.base_env = self.bb.env()
@@ -386,25 +388,23 @@ class Updater(object):
def commit_changes(self, pkg_ctx):
fail = False
-
try:
pkg_ctx['patch_file'] = None
- if 'recipe' in pkg_ctx:
- I(" %s: Auto commit changes ..." % pkg_ctx['PN'])
- self.git.commit(pkg_ctx['recipe'].commit_msg, self.opts['author'])
+ I(" %s: Auto commit changes ..." % pkg_ctx['PN'])
+ self.git.commit(pkg_ctx['commit_msg'], self.opts['author'])
- stdout = self.git.create_patch(pkg_ctx['workdir'])
- pkg_ctx['patch_file'] = stdout.strip()
+ stdout = self.git.create_patch(pkg_ctx['workdir'])
+ pkg_ctx['patch_file'] = stdout.strip()
- if not pkg_ctx['patch_file']:
- msg = "Patch file not generated."
- E(" %s: %s\n %s" % (pkg_ctx['PN'], msg, stdout))
- pkg_ctx['error'] = Error(msg, stdout)
- fail = True
- else:
- I(" %s: Save patch in directory: %s." %
- (pkg_ctx['PN'], pkg_ctx['workdir']))
+ if not pkg_ctx['patch_file']:
+ msg = "Patch file not generated."
+ E(" %s: %s\n %s" % (pkg_ctx['PN'], msg, stdout))
+ pkg_ctx['error'] = Error(msg, stdout)
+ fail = True
+ else:
+ I(" %s: Save patch in directory: %s." %
+ (pkg_ctx['PN'], pkg_ctx['workdir']))
except Error as e:
msg = ''
@@ -568,7 +568,7 @@ class Updater(object):
for step, msg in upgrade_steps:
if msg is not None:
I(" %s: %s" % (pkg_ctx['PN'], msg))
- step(self.bb, self.git, self.opts, pkg_ctx)
+ step(self.devtool, self.bb, self.git, self.opts, pkg_ctx)
succeeded_pkgs_ctx.append(pkg_ctx)
I(" %s: Upgrade SUCCESSFUL! Please test!" % pkg_ctx['PN'])
--
2.15.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [auh][PATCH 07/20] buildhistory.py: replace cleanall with cleansstate
2017-12-14 16:35 [auh][PATCH 00/20] Porting AUH to devtool and making it suitable for local use Alexander Kanavin
` (5 preceding siblings ...)
2017-12-14 16:35 ` [auh][PATCH 06/20] Add devtool support for upgrading recipes Alexander Kanavin
@ 2017-12-14 16:35 ` Alexander Kanavin
2017-12-14 16:35 ` [auh][PATCH 08/20] testimage.py: do not manipulate branches Alexander Kanavin
` (12 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Alexander Kanavin @ 2017-12-14 16:35 UTC (permalink / raw)
To: yocto
From: Alexander Kanavin <alex.kanavin@gmail.com>
It should not be really necessary, and wastes time
as the upstream source needs to be re-fetched.
Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
---
modules/buildhistory.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/modules/buildhistory.py b/modules/buildhistory.py
index 9085233..e78da05 100644
--- a/modules/buildhistory.py
+++ b/modules/buildhistory.py
@@ -49,7 +49,7 @@ class BuildHistory(object):
os.environ["BUILDHISTORY_DIR"] = self.buildhistory_dir
def init(self, machines):
- self.bb.cleanall(self.pn)
+ self.bb.cleansstate(self.pn)
for machine in machines:
self.bb.complete(self.pn, machine)
self.revs.append(self.git.last_commit("master"))
--
2.15.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [auh][PATCH 08/20] testimage.py: do not manipulate branches
2017-12-14 16:35 [auh][PATCH 00/20] Porting AUH to devtool and making it suitable for local use Alexander Kanavin
` (6 preceding siblings ...)
2017-12-14 16:35 ` [auh][PATCH 07/20] buildhistory.py: replace cleanall with cleansstate Alexander Kanavin
@ 2017-12-14 16:35 ` Alexander Kanavin
2017-12-14 16:35 ` [auh][PATCH 09/20] Replace references to Aníbal Limón as the maintainer Alexander Kanavin
` (11 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Alexander Kanavin @ 2017-12-14 16:35 UTC (permalink / raw)
To: yocto
From: Alexander Kanavin <alex.kanavin@gmail.com>
AUH simply creates a series of commits now, so we can create and
run testimage directly on the branch we're on.
Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
---
modules/testimage.py | 32 --------------------------------
1 file changed, 32 deletions(-)
diff --git a/modules/testimage.py b/modules/testimage.py
index da5c433..23d38d4 100644
--- a/modules/testimage.py
+++ b/modules/testimage.py
@@ -75,33 +75,6 @@ class TestImage():
return ' '.join(pkgs_out)
- def prepare_branch(self, pkgs_ctx):
- ok = False
-
- try:
- self.git.reset_hard()
- self.git.checkout_branch("master")
-
- try:
- self.git.delete_branch("testimage")
- except Error:
- pass
-
- self.git.create_branch("testimage")
- for c in pkgs_ctx:
- patch_file = os.path.join(c['workdir'], c['patch_file'])
- try:
- self.git.apply_patch(patch_file)
- except Exception as e:
- c['integration_error'] = e
- self.git.abort_patch()
-
- ok = True
- except Exception as e:
- self._log_error(" testimage: Failed to prepare branch.")
-
- return ok
-
def _parse_ptest_log(self, log_file):
ptest_results = {}
@@ -284,8 +257,6 @@ class TestImage():
self.pkgs_ctx.remove(pkg_ctx)
- if not self.prepare_branch(self.pkgs_ctx):
- handled = False
else:
handled = False
@@ -296,9 +267,6 @@ class TestImage():
I(" Testimage was enabled but any upgrade was successful.")
return
- if not self.prepare_branch(self.pkgs_ctx):
- return
-
I(" Images will test for %s." % ', '.join(self.opts['machines']))
for machine in self.opts['machines']:
I(" Testing images for %s ..." % machine)
--
2.15.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [auh][PATCH 09/20] Replace references to Aníbal Limón as the maintainer.
2017-12-14 16:35 [auh][PATCH 00/20] Porting AUH to devtool and making it suitable for local use Alexander Kanavin
` (7 preceding siblings ...)
2017-12-14 16:35 ` [auh][PATCH 08/20] testimage.py: do not manipulate branches Alexander Kanavin
@ 2017-12-14 16:35 ` Alexander Kanavin
2017-12-14 16:35 ` [auh][PATCH 10/20] modules/recipe: remove Alexander Kanavin
` (10 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Alexander Kanavin @ 2017-12-14 16:35 UTC (permalink / raw)
To: yocto
From: Alexander Kanavin <alex.kanavin@gmail.com>
Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
---
README | 2 +-
upgradehelper.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/README b/README
index e6b0dc2..60dbef9 100644
--- a/README
+++ b/README
@@ -163,7 +163,7 @@ The latest version of the code can always be found here:
Contributions are welcome. Please send patches / pull requests to
yocto@yoctoproject.org with '[auh]' in the subject also CC the
-current maintainer: Aníbal Limón <anibal.limon@linux.intel.com>.
+current maintainer: Alex Kanavin <alex.kanavin@gmail.com>.
License
-------
diff --git a/upgradehelper.py b/upgradehelper.py
index 5149993..9fa79b4 100755
--- a/upgradehelper.py
+++ b/upgradehelper.py
@@ -321,7 +321,7 @@ class Updater(object):
mail_footer = \
"Attached are the patch, license diff (if change) and bitbake log.\n" \
- "Any problem please contact Anibal Limon <anibal.limon@intel.com>.\n\n" \
+ "Any problem please file a bug at https://bugzilla.yoctoproject.org/enter_bug.cgi?product=Automated%20Update%20Handler\n\n" \
"Regards,\nThe Upgrade Helper"
if pkg_ctx['MAINTAINER'] in maintainer_override:
--
2.15.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [auh][PATCH 10/20] modules/recipe: remove
2017-12-14 16:35 [auh][PATCH 00/20] Porting AUH to devtool and making it suitable for local use Alexander Kanavin
` (8 preceding siblings ...)
2017-12-14 16:35 ` [auh][PATCH 09/20] Replace references to Aníbal Limón as the maintainer Alexander Kanavin
@ 2017-12-14 16:35 ` Alexander Kanavin
2017-12-14 16:35 ` [auh][PATCH 11/20] upgradehelper.py: remove the package ordering code Alexander Kanavin
` (9 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Alexander Kanavin @ 2017-12-14 16:35 UTC (permalink / raw)
To: yocto
From: Alexander Kanavin <alex.kanavin@gmail.com>
The hand-crafted recipe updating/rewriting has been replaced by calling devtool.
Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
---
modules/recipe/__init__.py | 0
modules/recipe/base.py | 724 ---------------------------------------------
modules/recipe/git.py | 98 ------
modules/recipe/svn.py | 28 --
4 files changed, 850 deletions(-)
delete mode 100644 modules/recipe/__init__.py
delete mode 100644 modules/recipe/base.py
delete mode 100644 modules/recipe/git.py
delete mode 100644 modules/recipe/svn.py
diff --git a/modules/recipe/__init__.py b/modules/recipe/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/modules/recipe/base.py b/modules/recipe/base.py
deleted file mode 100644
index 3798577..0000000
--- a/modules/recipe/base.py
+++ /dev/null
@@ -1,724 +0,0 @@
-#!/usr/bin/env python
-# vim: set ts=4 sw=4 et:
-#
-# Copyright (c) 2013 - 2014 Intel Corporation
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# AUTHORS
-# Laurentiu Palcu <laurentiu.palcu@intel.com>
-# Marius Avram <marius.avram@intel.com>
-#
-
-import os
-import re
-import sys
-import logging as log
-from logging import debug as D
-from logging import info as I
-from logging import warning as W
-
-from errors import *
-from utils.bitbake import *
-
-def is_recipe_or_include_file(env, full_path_f, f):
- is_file = os.path.isfile(full_path_f)
-
- is_recipe = f.find(env['PN']) == 0 and \
- f.find(env['PKGV']) != -1 and \
- f.find(".bb") != -1
-
- is_include = f.find(env['PN']) == 0 and \
- f.find(".inc") != -1
-
- return is_file and (is_recipe or is_include)
-
-def modify_recipe_files(func):
- def modify(env, recipe_dir, *args, **kwargs):
- for f in os.listdir(recipe_dir):
- full_path_f = os.path.join(recipe_dir, f)
- if is_recipe_or_include_file(env, full_path_f, f):
- with open(full_path_f + ".tmp", "w+") as temp_recipe:
- with open(full_path_f) as recipe:
- for line in recipe:
- func(line, temp_recipe, *args, **kwargs)
- os.rename(full_path_f + ".tmp", full_path_f)
- return modify
-
-def read_recipe_files(func):
- def read(env, recipe_dir):
- for f in os.listdir(recipe_dir):
- full_path_f = os.path.join(recipe_dir, f)
- if is_recipe_or_include_file(env, full_path_f, f):
- with open(full_path_f) as recipe:
- for line in recipe:
- func(line)
- return read
-
-class Recipe(object):
- def __init__(self, env, new_ver, interactive, workdir, recipe_dir, bitbake, git):
- self.env = env
- self.new_ver = new_ver
- self.interactive = interactive
- self.workdir = workdir
- self.recipe_dir = recipe_dir
- self.bb = bitbake
- self.bb.set_log_dir(workdir)
- self.git = git
-
- self.retried_recipes = set()
- self.license_diff_file = None
-
- self.recipes_renamed = False
- self.checksums_changed = False
-
- self.removed_patches = False
-
- self.suffixes = [
- "tar.gz", "tgz", "zip", "tar.bz2", "tar.xz", "tar.lz4", "bz2",
- "lz4", "orig.tar.gz", "src.tar.gz", "src.rpm", "src.tgz",
- "svnr\d+.tar.bz2", "stable.tar.gz", "src.rpm"]
- self.old_env = None
-
- self.commit_msg = self.env['PN'] + ": upgrade to " + self.new_ver + "\n\n"
- self.comment_patches_msg = "\n\nCommented the following patch(es):\n"
-
- self._inherits = None
-
- super(Recipe, self).__init__()
-
- def get_inherits(self):
- @read_recipe_files
- def _get_inherits(line):
- m = re.search("^inherit (.*)$", line)
- if m:
- tmp = m.group(1).split()
- self._inherits.extend(tmp)
-
- if not self._inherits:
- self._inherits = []
- _get_inherits(self.env, self.recipe_dir)
-
- return self._inherits
-
- def update_env(self, env):
- self.env = env
-
- def _rename_files_dir(self, old_ver, new_ver):
- # The files directory is renamed only if the previous
- # one has the following format PackageName-PackageVersion.
- # Otherwise is kept the same way.
- src_dir = os.path.join(self.recipe_dir, self.env['PN'] + "-" + old_ver)
- dest_dir = os.path.join(self.recipe_dir, self.env['PN'] + "-" + new_ver)
-
- if os.path.exists(src_dir) and os.path.isdir(src_dir):
- self.git.mv(src_dir, dest_dir)
-
- def rename(self):
- # clean PR before renaming
- @modify_recipe_files
- def _clean_pr(line, temp_recipe, *args, **kwargs):
- if not (line.startswith("PR=") or line.startswith("PR =")):
- temp_recipe.write(line)
- _clean_pr(self.env, self.recipe_dir)
-
- # rename recipes (not directories)
- for path in os.listdir(self.recipe_dir):
- full_path = os.path.join(self.recipe_dir, path)
- if os.path.isfile(full_path) \
- and path.find(self.env['PN']) == 0 \
- and path.find(self.env['PKGV']) != -1:
- new_path = re.sub(re.escape(self.env['PKGV']), self.new_ver, path)
- self.git.mv(os.path.join(self.recipe_dir, path),
- os.path.join(self.recipe_dir, new_path))
-
- # rename files/PN-PV directories to PN
- self._rename_files_dir(self.env['PKGV'], self.new_ver)
-
- self.recipes_renamed = True
-
- # since we did some renaming, backup the current environment
- self.old_env = self.env
-
- def create_diff_file(self, file, old_md5, new_md5):
- old_file = os.path.join(self.old_env['S'], file)
- new_file = os.path.join(self.env['S'], file)
- cmd = "diff -Nup " + old_file + " " + new_file + " > " + \
- os.path.join(self.workdir, os.path.basename(file + ".diff"))
-
- try:
- stdout, stderr = bb.process.run(cmd)
- except bb.process.ExecutionError:
- pass
-
- with open(os.path.join(self.workdir, "license_checksums.txt"), "w+") as f:
- f.write("old checksum = %s\n" % old_md5)
- f.write("new_checksum = %s\n" % new_md5)
-
- def _change_recipe_checksums(self, fetch_log):
- sums = {}
-
- with open(os.path.realpath(fetch_log)) as log:
- for line in log:
- m = None
- key = None
- m1 = re.match("^SRC_URI\[(.*)md5sum\].*", line)
- m2 = re.match("^SRC_URI\[(.*)sha256sum\].*", line)
- if m1:
- m = m1
- key = "md5sum"
- elif m2:
- m = m2
- key = "sha256sum"
-
- if m:
- name = m.group(1)
- sum_line = m.group(0) + '\n'
- if name not in sums:
- sums[name] = {}
- sums[name][key] = sum_line;
-
- if len(sums) == 0:
- raise FetchError()
-
- # checksums are usually in the main recipe but they can also be in inc
- # files... Go through the recipes/inc files until we find them
- @modify_recipe_files
- def _update_recipe_checksums(line, temp_recipe, *args, **kwargs):
- sums = args[0]
- for name in sums:
- m1 = re.match("^SRC_URI\["+ name + "md5sum\].*", line)
- m2 = re.match("^SRC_URI\["+ name + "sha256sum\].*", line)
- if m1:
- temp_recipe.write(sums[name]["md5sum"])
- elif m2:
- temp_recipe.write(sums[name]["sha256sum"])
- else:
- temp_recipe.write(line)
-
- I(" %s: Update recipe checksums ..." % self.env['PN'])
- _update_recipe_checksums(self.env, self.recipe_dir, sums)
-
- self.checksums_changed = True
-
- def _is_uri_failure(self, fetch_log):
- uri_failure = None
- checksum_failure = None
- with open(os.path.realpath(fetch_log)) as log:
- for line in log:
- if not uri_failure:
- uri_failure = re.match(".*Fetcher failure for URL.*", line)
- if not checksum_failure:
- checksum_failure = re.match(".*Checksum mismatch.*", line)
- if uri_failure and not checksum_failure:
- return True
- else:
- return False
-
-
- def _change_source_suffix(self, new_suffix):
- # Will change the extension of the archive from the SRC_URI
-
- @modify_recipe_files
- def _change(line, temp_recipe, *args, **kwargs):
- d = args[0]
-
- # source on first line
- m1 = re.match("^SRC_URI.*\${PV}\.(.*)[\" \\\\].*", line)
- # SRC_URI alone on the first line
- m2 = re.match("^SRC_URI.*", line)
- # source on second line
- m3 = re.match(".*\${PV}\.(.*)[\" \\\\].*", line)
- if m1:
- old_suffix = m1.group(1)
- line = line.replace(old_suffix, new_suffix+" ")
- if m2 and not m1:
- d['source_found'] = True
- if m3 and d['source_found']:
- old_suffix = m3.group(1)
- line = line.replace(old_suffix, new_suffix+" ")
- d['source_found'] = False
-
- temp_recipe.write(line)
-
- d = {}
- d['source_found'] = False
- _change(self.env, self.recipe_dir, d)
-
- def _comment_patch_uri(self, uri):
- @modify_recipe_files
- def _comment(line, temp_recipe, *args, **kwargs):
- d = args[0]
- uri = d['uri']
-
- m1 = re.match("SRC_URI *\+*= *\" *" + uri + " *\"", line)
- m2 = re.match("(SRC_URI *\+*= *\" *)" + uri + " *\\\\", line)
- m3 = re.match("[\t ]*" + uri + " *\\\\", line)
- m4 = re.match("([\t ]*)" + uri + " *\"", line)
-
- if m1 or m2 or m3 or m4:
- d['commented'] = True
-
- if not d['start']:
- d['start'] = True
-
- # patch on a single SRC_URI line:
- if m1:
- d['patches'].append(line)
- d['end'] = True
- # patch is on the first SRC_URI line
- elif m2:
- d['patches'].append(line)
- temp_recipe.write(m2.group(1) + "\\\n")
- # patch is in the middle
- elif m3:
- d['patches'].append(line)
- # patch is last in list
- elif m4:
- d['end'] = True
- d['patches'].append(line)
- temp_recipe.write(m4.group(1) + "\"\n")
- else:
- temp_recipe.write(line)
-
- if d['start'] and (d['end'] or line.strip().startswith('"')):
- if d['patches']:
- for p in d['patches']:
- line = p.rstrip()
- if line.endswith('\\'):
- line = line[:-1]
- temp_recipe.write("#%s\n" % line)
-
- d['start'] = False
- d['end'] = False
-
- d = {}
- d['commented'] = False
-
- d['uri'] = uri
- d['patches'] = []
- d['start'] = False
- d['end'] = False
- _comment(self.env, self.recipe_dir, d)
-
- return d['commented']
-
- def _comment_faulty_patch(self, patch_log):
- patch_file = None
- is_reverse_applied = False
-
- with open(patch_log) as log:
- for line in log:
- m1 = re.match("^Patch ([^ ]*) does not apply.*", line)
- m2 = re.match("Patch ([^ ]*) can be reverse-applied", line)
- if m2:
- m1 = m2
- is_reverse_applied = True
- if m1:
- patch_file = m1.group(1)
- break
-
- if not patch_file:
- return False
-
- I(" %s: Commenting patch %s ..." % (self.env['PN'], patch_file))
- reason = None
- found = False
- dirs = [self.env['PN'] + "-" + self.env['PKGV'], self.env['PN'], "files"]
- for dir in dirs:
- patch_file_path = os.path.join(self.recipe_dir, dir, patch_file)
- if not os.path.exists(patch_file_path):
- continue
- else:
- found = True
-
- # Find out upstream status of the patch
- with open(patch_file_path) as patch:
- for line in patch:
- m = re.match(".*Upstream-Status:(.*)\n", line)
- if m:
- reason = m.group(1).strip().split()[0].lower()
-
- if not self._comment_patch_uri("file://" + patch_file):
- return False
- if not found:
- return False
-
- self.comment_patches_msg += " * " + patch_file
- if reason:
- self.comment_patches_msg += " (" + reason + ") "
- if is_reverse_applied:
- self.comment_patches_msg += "+ reverse-applied"
- self.comment_patches_msg += "\n"
-
- return True
-
- def _is_license_issue(self, config_log):
- with open(config_log) as log:
- for line in log:
- m = re.match("ERROR: " + self.env['PN'] +
- "[^:]*: md5 data is not matching for file", line)
- if m is not None:
- return True
-
- return False
-
- def _license_issue_handled(self, config_log):
- @modify_recipe_files
- def _update_license_checksum(line, temp_recipe, *args, **kwargs):
- d = args[0]
- m = re.match("(.*)" + d['old_md5'] + "(.*)", line)
- if m is not None:
- temp_recipe.write(m.group(1) + d['new_md5'] + m.group(2) + "\n")
- else:
- temp_recipe.write(line)
-
- license_file = None
- with open(config_log) as log:
- for line in log:
- if not line.startswith("ERROR:"):
- continue
- m_old = re.match("ERROR: " + self.env['PN'] +
- "[^:]*: md5 data is not matching for file://([^;]*);md5=(.*)$", line)
- if not m_old:
- m_old = re.match("ERROR: " + self.env['PN'] +
- "[^:]*: md5 data is not matching for file://([^;]*);beginline=[0-9]*;endline=[0-9]*;md5=(.*)$", line)
- if not m_old:
- m_old = re.match("ERROR: " + self.env['PN'] +
- "[^:]*: md5 data is not matching for file://([^;]*);endline=[0-9]*;md5=(.*)$", line)
- if not m_old:
- m_old = re.match("ERROR: " + self.env['PN'] +
- "[^:]*: md5 data is not matching for file://([^;]*);beginline=[0-9]*;md5=(.*)$", line)
- m_new = re.match("ERROR: " + self.env['PN'] +
- "[^:]*: The new md5 checksum is (.*)", line)
- if m_old:
- license_file = m_old.group(1)
- old_md5 = m_old.group(2)
- elif m_new:
- new_md5 = m_new.group(1)
-
- if license_file is not None:
- d = {}
- d['old_md5'] = old_md5
- d['new_md5'] = new_md5
- _update_license_checksum(self.env, self.recipe_dir, d)
-
- self.create_diff_file(license_file, old_md5, new_md5)
- self.license_diff_file = os.path.join(self.workdir, os.path.basename(license_file + ".diff"))
-
- if self.interactive:
- W(" %s: license checksum failed for file %s. The recipe has"
- "been updated! View diff? (Y/n)" % (self.env['PN'], license_file))
- answer = sys.stdin.readline().strip().upper()
- if answer == '' or answer == 'Y':
- I(" ################ Licence file diff #################")
- with open(self.license_diff_file) as diff:
- I("%s" % diff.read())
- I(" ####################################################")
- I(" Retry compilation? (Y/n)")
- answer = sys.stdin.readline().strip().upper()
- if answer == '' or answer == 'Y':
- return True
- else:
- W(" %s: license checksum failed for file %s."
- " The recipe has been updated! Diff file located at %s" %
- (self.env['PN'], license_file, self.license_diff_file))
- I(" Recompiling ...")
- self.commit_msg += "License checksum changed for file " + license_file
- return True
-
- return False
-
- def get_license_diff_file_name(self):
- file_name = None
- if not self.license_diff_file is None:
- file_name = os.path.basename(self.license_diff_file)
-
- return file_name
-
- def _get_failed_recipes(self, output):
- failed_tasks = dict()
- machine = None
-
- for line in output.split("\n"):
- machine_match = re.match("MACHINE[\t ]+= *\"(.*)\"$", line)
- task_log_match = re.match("ERROR: Logfile of failure stored in: (.*/([^/]*)/[^/]*/temp/log\.(.*)\.[0-9]*)", line)
- # For some reason do_package is reported differently
- qa_issue_match = re.match("ERROR: QA Issue: ([^ :]*): (.*) not shipped", line)
-
- if task_log_match:
- failed_tasks[task_log_match.group(2)] = (task_log_match.group(3), task_log_match.group(1))
- elif qa_issue_match:
- # Improvise path to log file
- failed_tasks[qa_issue_match.group(1)] = ("do_package", self.bb.get_stdout_log())
- elif machine_match:
- machine = machine_match.group(1)
-
- # we didn't detect any failed tasks? then something else is wrong
- if len(failed_tasks) == 0:
- raise Error("could not detect failed task")
-
- return (machine, failed_tasks)
-
- def _is_incompatible_host(self, output):
- for line in output.split("\n"):
- incomp_host = re.match("ERROR: " + self.env['PN'] + " was skipped: incompatible with host (.*) \(.*$", line)
-
- if incomp_host is not None:
- return True
-
- return False
-
- def _add_not_shipped(self, package_log):
- files_not_shipped = False
- files = []
- occurences = []
- prefixes = {
- "/usr" : "prefix",
- "/bin" : "base_bindir",
- "/sbin" : "base_sbindir",
- "/lib" : "base_libdir",
- "/usr/share" : "datadir",
- "/etc" : "sysconfdir",
- "/var" : "localstatedir",
- "/usr/share/info" : "infodir",
- "/usr/share/man" : "mandir",
- "/usr/share/doc" : "docdir",
- "/srv" : "servicedir",
- "/usr/bin" : "bindir",
- "/usr/sbin" : "sbindir",
- "/usr/libexec" : "libexecdir",
- "/usr/lib" : "libdir",
- "/usr/include" : "includedir",
- "/usr/lib/opie" : "palmtopdir",
- "/usr/lib/opie" : "palmqtdir",
- }
-
- with open(package_log) as log:
- for line in log:
- if re.match(".*Files/directories were installed but not shipped.*", line):
- I(" %s: Add new files in recipe ..." % self.env['PN'])
- files_not_shipped = True
- # Extract path
- line = line.strip()
- if line:
- line = line.split()[0]
- if files_not_shipped and os.path.isabs(line):
- # Count occurences for globbing
- path_exists = False
- for i in range(0, len(files)):
- if line.find(files[i]) == 0:
- path_exists = True
- occurences[i] += 1
- break
- if not path_exists:
- files.append(line)
- occurences.append(1)
-
- for i in range(0, len(files)):
- # Change paths to globbing expressions where is the case
- if occurences[i] > 1:
- files[i] += "/*"
- largest_prefix = ""
- # Substitute prefix
- for prefix in prefixes:
- if files[i].find(prefix) == 0 and len(prefix) > len(largest_prefix):
- largest_prefix = prefix
- if largest_prefix:
- replacement = "${" + prefixes[largest_prefix] + "}"
- files[i] = files[i].replace(largest_prefix, replacement)
-
- @modify_recipe_files
- def _append_new_files(line, temp_recipe, *args, **kwargs):
- d = args[0]
-
- if re.match("^FILES_\${PN}[ +=].*", line):
- d['files_clause'] = True
- temp_recipe.write(line)
- return
-
- # Get front spacing
- if d['files_clause']:
- front_spacing = re.sub("[^ \t]", "", line)
-
- # Append once the last line has of FILES has been reached
- if re.match(".*\".*", line) and d['files_clause']:
- d['files_clause'] = False
- line = line.replace("\"", "")
- line = line.rstrip()
- front_spacing = re.sub("[^ \t]", "", line)
-
- # Do not write an empty line
- if line.strip():
- temp_recipe.write(line + " \\\n")
-
- # Add spacing in case there was none
- if len(front_spacing) == 0:
- front_spacing = " " * 8
-
- # Write to file
- for i in range(len(files) - 1):
- line = front_spacing + files[i] + " \\\n"
- temp_recipe.write(line)
-
- line = front_spacing + files[len(files) - 1] + "\"\n"
- temp_recipe.write(line)
- return
-
- temp_recipe.write(line)
-
- d = {}
- d['files_clause'] = False
- _append_new_files(self.env, self.recipe_dir, d)
-
- return files_not_shipped
-
- def unpack(self):
- self.bb.unpack(self.env['PN'])
-
- def fetch(self):
- from recipe.git import GitRecipe
-
- def _try_fetch():
- try:
- self.bb.fetch(self.env['PN'])
- return
- except Error as e:
- machine, failed_recipes = self._get_failed_recipes(e.stdout)
- if not self.env['PN'] in failed_recipes:
- raise Error("Unknown error occured during fetch",
- stdout = e.stdout, stderr = e.stderr)
-
- fetch_log = failed_recipes[self.env['PN']][1]
-
- if not self._is_uri_failure(fetch_log) and not \
- self.checksums_changed:
- self._change_recipe_checksums(fetch_log)
- self.checksums_changed = True
- return True
-
- return False
-
- succeed = _try_fetch()
-
- if not succeed and not isinstance(self, GitRecipe):
- for sfx in self.suffixes:
- I(" Trying new SRC_URI suffix: %s ..." % sfx)
- self._change_source_suffix(sfx)
-
- succeed = _try_fetch()
- if succeed:
- break
-
- if not succeed:
- raise Error("Can't built a valid SRC_URI")
- elif self.recipes_renamed and not self.checksums_changed:
- raise Error("Fetch succeeded without changing checksums")
-
- def cleanall(self):
- self.bb.cleanall(self.env['PN'])
-
- def _clean_failed_recipes(self, failed_recipes):
- already_retried = False
- for recipe in failed_recipes:
- if recipe in self.retried_recipes:
- # we already retried, we'd best leave it to a human to handle
- # it :)
- already_retried = True
- # put the recipe in the retried list
- self.retried_recipes.add(recipe)
-
- if already_retried:
- return False
- else:
- I(" %s: The following recipe(s): %s, failed. "
- "Doing a 'cleansstate' and then retry ..." %
- (self.env['PN'], ' '.join(failed_recipes.keys())))
-
- self.bb.cleansstate(' '.join(failed_recipes.keys()))
- return True
-
- def _undo_temporary(self):
- # Undo removed patches
- if self.removed_patches:
- self.git.checkout_branch("upgrades")
- self.git.delete_branch("comment_patches")
- self.git.reset_hard()
- self.git.reset_soft(1)
- self.removed_patches = False
-
- def compile(self, machine):
- try:
- self.bb.complete(self.env['PN'], machine)
- if self.removed_patches:
- # move temporary changes into upgrades branch
- self.git.checkout_branch("upgrades")
- self.git.delete_branch("comment_patches")
- self.git.reset_soft(1)
- self.commit_msg += self.comment_patches_msg + "\n"
- self.removed_patches = False
- except Error as e:
- if self._is_incompatible_host(e.stdout):
- W(" %s: compilation failed: incompatible host" % self.env['PN'])
- return
- machine, failed_recipes = self._get_failed_recipes(e.stdout)
- if not self.env['PN'] in failed_recipes:
- if not self._clean_failed_recipes(failed_recipes):
- self._undo_temporary()
- raise CompilationError()
-
- # retry
- self.compile(machine)
- else:
- failed_task = failed_recipes[self.env['PN']][0]
- log_file = failed_recipes[self.env['PN']][1]
- if failed_task == "do_patch":
-
- # Comment one patch after the other until
- # compilation works.
- if not self.removed_patches:
- self.git.commit("temporary")
- self.git.create_branch("comment_patches")
- self.git.checkout_branch("comment_patches")
- self.removed_patches = True
-
- if not self._comment_faulty_patch(log_file):
- self._undo_temporary()
- raise PatchError()
-
- # retry
- I(" %s: Recompiling for %s ..." % (self.env['PN'], machine))
- self.compile(machine)
- elif failed_task == "do_configure":
- self._undo_temporary()
- if not self._is_license_issue(log_file):
- raise ConfigureError()
-
- if not self._license_issue_handled(log_file):
- raise LicenseError()
- #retry
- self.compile(machine)
- elif failed_task == "do_fetch":
- raise FetchError()
- elif failed_task == "do_package":
- raise PackageError()
- #if self._add_not_shipped(log_file):
- # self.compile(machine)
- #else:
- else:
- self._undo_temporary()
- # throw a compilation exception for everything else. It
- # doesn't really matter
- raise CompilationError()
diff --git a/modules/recipe/git.py b/modules/recipe/git.py
deleted file mode 100644
index 95e8810..0000000
--- a/modules/recipe/git.py
+++ /dev/null
@@ -1,98 +0,0 @@
-#!/usr/bin/env python
-# vim: set ts=4 sw=4 et:
-#
-# Copyright (c) 2013 - 2014 Intel Corporation
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# AUTHORS
-# Laurentiu Palcu <laurentiu.palcu@intel.com>
-# Marius Avram <marius.avram@intel.com>
-#
-
-import re
-import os
-
-from errors import *
-from recipe.base import Recipe
-
-class GitRecipe(Recipe):
- def _extract_tag_from_ver(self, ver):
- m = re.match("(.*)\+.*\+.*", ver)
- if m is not None:
- return m.group(1)
-
- # allow errors in the reporting system
- return ver
-
- def _get_tag_sha1(self, new_tag):
- m = re.match(".*(git://[^ ;]*).*", self.env['SRC_URI'])
- if m is None:
- raise Error("could not extract repo url from SRC_URI")
-
- repo_url = m.group(1)
- tags = self.git.ls_remote(repo_url, "--tags")
-
- # Try to find tag ending with ^{}
- for tag in tags.split('\n'):
- if tag.endswith(new_tag + "^{}"):
- return tag.split()[0]
-
- # If not found, try to find simple tag
- for tag in tags.split('\n'):
- if tag.endswith(new_tag):
- return tag.split()[0]
-
- return None
-
- def rename(self):
- old_git_tag = self._extract_tag_from_ver(self.env['PKGV'])
- new_git_tag = self._extract_tag_from_ver(self.new_ver)
-
- if new_git_tag == old_git_tag:
- raise UpgradeNotNeededError()
-
- tag_sha1 = self._get_tag_sha1(new_git_tag)
- if tag_sha1 is None:
- raise Error("could not extract tag sha1")
-
- for f in os.listdir(self.recipe_dir):
- full_path_f = os.path.join(self.recipe_dir, f)
- if os.path.isfile(full_path_f) and \
- ((f.find(self.env['PN']) == 0 and (f.find(old_git_tag) != -1 or
- f.find("git") != -1) and f.find(".bb") != -1) or
- (f.find(self.env['PN']) == 0 and f.find(".inc") != -1)):
- with open(full_path_f + ".tmp", "w+") as temp_recipe:
- with open(full_path_f) as recipe:
- for line in recipe:
- m1 = re.match("^SRCREV *= *\".*\"", line)
- m2 = re.match("PV *= *\"[^\+]*(.*)\"", line)
- if m1 is not None:
- temp_recipe.write("SRCREV = \"" + tag_sha1 + "\"\n")
- elif m2 is not None:
- temp_recipe.write("PV = \"" + new_git_tag + m2.group(1) + "\"\n")
- else:
- temp_recipe.write(line)
-
- os.rename(full_path_f + ".tmp", full_path_f)
-
- self.env['PKGV'] = old_git_tag
- self.new_ver = new_git_tag
-
- super(GitRecipe, self).rename()
-
- def fetch(self):
- pass
-
diff --git a/modules/recipe/svn.py b/modules/recipe/svn.py
deleted file mode 100644
index 6dadafd..0000000
--- a/modules/recipe/svn.py
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/usr/bin/env python
-# vim: set ts=4 sw=4 et:
-#
-# Copyright (c) 2013 - 2014 Intel Corporation
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# AUTHORS
-# Laurentiu Palcu <laurentiu.palcu@intel.com>
-# Marius Avram <marius.avram@intel.com>
-#
-
-from recipe.base import Recipe
-
-class SvnRecipe(Recipe):
- pass
--
2.15.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [auh][PATCH 11/20] upgradehelper.py: remove the package ordering code
2017-12-14 16:35 [auh][PATCH 00/20] Porting AUH to devtool and making it suitable for local use Alexander Kanavin
` (9 preceding siblings ...)
2017-12-14 16:35 ` [auh][PATCH 10/20] modules/recipe: remove Alexander Kanavin
@ 2017-12-14 16:35 ` Alexander Kanavin
2017-12-14 16:35 ` [auh][PATCH 12/20] utils/git.py: print current dir when git fails Alexander Kanavin
` (8 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Alexander Kanavin @ 2017-12-14 16:35 UTC (permalink / raw)
To: yocto
From: Alexander Kanavin <alex.kanavin@gmail.com>
It was broken and commented out, and not really necessary.
Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
---
upgradehelper.py | 80 --------------------------------------------------------
1 file changed, 80 deletions(-)
diff --git a/upgradehelper.py b/upgradehelper.py
index 9fa79b4..d62c9b2 100755
--- a/upgradehelper.py
+++ b/upgradehelper.py
@@ -439,87 +439,7 @@ class Updater(object):
else:
W("No recipes attempted, not sending status mail!")
- def _order_pkgs_to_upgrade(self, pkgs_to_upgrade):
- def _get_pn_dep_dic(pn_list, dependency_file):
- import re
-
- pn_dep_dic = {}
-
- with open(dependency_file) as dep:
- data = dep.read()
- dep.close()
-
- for line in data.split('\n'):
- m = re.search('^"(.*)" -> "(.*)"$', line)
- if not m:
- continue
-
- pn = m.group(1)
- pn_dep = m.group(2)
- if pn == pn_dep:
- continue
-
- if pn in pn_list:
- if pn_dep in pn_list:
- if pn in pn_dep_dic.keys():
- pn_dep_dic[pn].append(pn_dep)
- else:
- pn_dep_dic[pn] = [pn_dep]
- elif not pn in pn_dep_dic.keys():
- pn_dep_dic[pn] = []
-
- return pn_dep_dic
-
- def _dep_resolve(graph, node, resolved, seen):
- seen.append(node)
-
- for edge in graph[node]:
- if edge not in resolved:
- if edge in seen:
- raise RuntimeError("Packages %s and %s have " \
- "a circular dependency." \
- % (node, edge))
- _dep_resolve(graph, edge, resolved, seen)
-
- resolved.append(node)
-
-
- pn_list = []
- for pn, new_ver, maintainer in pkgs_to_upgrade:
- pn_list.append(pn)
-
- try:
- self.bb.dependency_graph(' '.join(pn_list))
- except Error as e:
- multiple_providers = False
- for l in e.stdout.split('\n'):
- if l.find("ERROR: Multiple .bb files are due to be built which each provide") == 0:
- multiple_providers = True
- if not multiple_providers:
- raise e
-
- dependency_file = os.path.join(get_build_dir(), "pn-depends.dot")
-
- pkgs_to_upgrade_ordered = []
- pn_list_ordered = []
-
- pn_dep_dic = _get_pn_dep_dic(pn_list, dependency_file)
- if pn_dep_dic:
- root = "__root_node__"
- pn_dep_dic[root] = pn_dep_dic.keys()
- _dep_resolve(pn_dep_dic, root, pn_list_ordered, [])
- pn_list_ordered.remove(root)
-
- for pn_ordered in pn_list_ordered:
- for pn, new_ver, maintainer in pkgs_to_upgrade:
- if pn == pn_ordered:
- pkgs_to_upgrade_ordered.append([pn, new_ver, maintainer])
-
- return pkgs_to_upgrade_ordered
-
def run(self, package_list=None):
- #pkgs_to_upgrade = self._order_pkgs_to_upgrade(
- # self._get_packages_to_upgrade(package_list))
pkgs_to_upgrade = self._get_packages_to_upgrade(package_list)
total_pkgs = len(pkgs_to_upgrade)
--
2.15.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [auh][PATCH 12/20] utils/git.py: print current dir when git fails
2017-12-14 16:35 [auh][PATCH 00/20] Porting AUH to devtool and making it suitable for local use Alexander Kanavin
` (10 preceding siblings ...)
2017-12-14 16:35 ` [auh][PATCH 11/20] upgradehelper.py: remove the package ordering code Alexander Kanavin
@ 2017-12-14 16:35 ` Alexander Kanavin
2017-12-14 16:35 ` [auh][PATCH 13/20] testimage.py: do not call into removed code Alexander Kanavin
` (7 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Alexander Kanavin @ 2017-12-14 16:35 UTC (permalink / raw)
To: yocto
From: Alexander Kanavin <alex.kanavin@gmail.com>
Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
---
modules/utils/git.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/modules/utils/git.py b/modules/utils/git.py
index d1fe41b..a6f5422 100644
--- a/modules/utils/git.py
+++ b/modules/utils/git.py
@@ -40,7 +40,7 @@ class Git(object):
try:
stdout, stderr = bb.process.run(cmd)
except bb.process.ExecutionError as e:
- D("%s returned:\n%s" % (cmd, e.__str__()))
+ D("%s executed from %s returned:\n%s" % (cmd, self.repo_dir, e.__str__()))
raise Error("The following git command failed: " + operation,
e.stdout, e.stderr)
--
2.15.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [auh][PATCH 13/20] testimage.py: do not call into removed code
2017-12-14 16:35 [auh][PATCH 00/20] Porting AUH to devtool and making it suitable for local use Alexander Kanavin
` (11 preceding siblings ...)
2017-12-14 16:35 ` [auh][PATCH 12/20] utils/git.py: print current dir when git fails Alexander Kanavin
@ 2017-12-14 16:35 ` Alexander Kanavin
2017-12-14 16:35 ` [auh][PATCH 14/20] testimage.py: add extra logging for the testimage logs retrieval Alexander Kanavin
` (6 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Alexander Kanavin @ 2017-12-14 16:35 UTC (permalink / raw)
To: yocto
From: Alexander Kanavin <alex.kanavin@gmail.com>
This was using the removed recipe parsing code to determine if
a package supports ptest; it should be replaced with using bitbake API
but for now we can simply add all update packages to the list.
Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
---
modules/testimage.py | 12 ++----------
1 file changed, 2 insertions(+), 10 deletions(-)
diff --git a/modules/testimage.py b/modules/testimage.py
index 23d38d4..c3352e5 100644
--- a/modules/testimage.py
+++ b/modules/testimage.py
@@ -53,15 +53,6 @@ class TestImage():
os.environ['BB_ENV_EXTRAWHITE'] = os.environ['BB_ENV_EXTRAWHITE'] + \
" TEST_SUITES CORE_IMAGE_EXTRA_INSTALL"
- def _get_ptest_pkgs(self, pkgs_ctx):
- pkgs = []
-
- for c in pkgs_ctx:
- if "ptest" in c['recipe'].get_inherits():
- pkgs.append(c)
-
- return pkgs
-
def _get_pkgs_to_install(self, pkgs, ptest=False):
pkgs_out = []
@@ -149,7 +140,8 @@ class TestImage():
def ptest(self, pkgs_ctx, machine):
image = 'core-image-minimal'
- ptest_pkgs = self._get_ptest_pkgs(pkgs_ctx)
+ # should use bitbake API here to trim down the list to only the recipes that inherit ptest
+ ptest_pkgs = pkgs_ctx
os.environ['CORE_IMAGE_EXTRA_INSTALL'] = \
self._get_pkgs_to_install(ptest_pkgs, ptest=True)
--
2.15.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [auh][PATCH 14/20] testimage.py: add extra logging for the testimage logs retrieval
2017-12-14 16:35 [auh][PATCH 00/20] Porting AUH to devtool and making it suitable for local use Alexander Kanavin
` (12 preceding siblings ...)
2017-12-14 16:35 ` [auh][PATCH 13/20] testimage.py: do not call into removed code Alexander Kanavin
@ 2017-12-14 16:35 ` Alexander Kanavin
2017-12-14 16:35 ` [auh][PATCH 15/20] upgradehelper.py: fixups to the email message template Alexander Kanavin
` (5 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Alexander Kanavin @ 2017-12-14 16:35 UTC (permalink / raw)
To: yocto
From: Alexander Kanavin <alex.kanavin@gmail.com>
This helped with diagnozing issues described in
https://bugzilla.yoctoproject.org/show_bug.cgi?id=12396
Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
---
modules/testimage.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/modules/testimage.py b/modules/testimage.py
index c3352e5..6c4ed2b 100644
--- a/modules/testimage.py
+++ b/modules/testimage.py
@@ -98,8 +98,10 @@ class TestImage():
if name in files:
result.append(os.path.join(root, name))
+ D("Found logs named %s for machine %s: %s" %(name, machine, result))
for ptest_log in result:
if machine in ptest_log:
+ D("Picked log: %s" %(ptest_log))
return ptest_log
def _get_failed_recipe(self, log):
--
2.15.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [auh][PATCH 15/20] upgradehelper.py: fixups to the email message template
2017-12-14 16:35 [auh][PATCH 00/20] Porting AUH to devtool and making it suitable for local use Alexander Kanavin
` (13 preceding siblings ...)
2017-12-14 16:35 ` [auh][PATCH 14/20] testimage.py: add extra logging for the testimage logs retrieval Alexander Kanavin
@ 2017-12-14 16:35 ` Alexander Kanavin
2017-12-14 16:35 ` [auh][PATCH 16/20] upgradehelper.py: send email even if recipe upgrade failed Alexander Kanavin
` (4 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Alexander Kanavin @ 2017-12-14 16:35 UTC (permalink / raw)
To: yocto
From: Alexander Kanavin <alex.kanavin@gmail.com>
Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
---
upgradehelper.py | 43 +++++++++++++++++++++++++++----------------
1 file changed, 27 insertions(+), 16 deletions(-)
diff --git a/upgradehelper.py b/upgradehelper.py
index d62c9b2..d3cc2c4 100755
--- a/upgradehelper.py
+++ b/upgradehelper.py
@@ -294,19 +294,20 @@ class Updater(object):
"to upgrade the recipe to *%s* has %s.\n\n"
license_change_info = \
- "*LICENSE CHANGED* please review the %s file and update the LICENSE\n" \
- "variable in the recipe if is needed.\n\n"
+ "*LICENSE CHANGED* please review the %s file, update the LICENSE\n" \
+ "variable in the recipe and summarize the changes in the commit message.\n\n"
next_steps_info = \
- "The recipe has been successfully compiled for machines %s.\n\n" \
"Next steps:\n" \
" - apply the patch: git am %s\n" \
- " - check that required upstream patches have not been commented from the recipe,\n" \
- " if upstream patches were commented the reason is specified in the commit message.\n" \
+ " - check the changes to upstream patches and summarize them in the commit message,\n" \
" - compile an image that contains the package\n" \
" - perform some basic sanity tests\n" \
" - amend the patch and sign it off: git commit -s --reset-author --amend\n" \
- " - send it to the list\n\n" \
+ " - send it to the appropriate mailing list\n\n" \
+ "Alternatively, if you believe the recipe should not be upgraded at this time,\n" \
+ "you can fill RECIPE_NO_UPDATE_REASON in respective recipe file so that\n" \
+ "automatic upgrades would no longer be attempted.\n\n"
testimage_integration_error = \
"The recipe *FAILED* in testimage integration. Attached is the log file.\n\n"
@@ -320,7 +321,7 @@ class Updater(object):
"the next machines %s. Attached is the log file.\n\n" \
mail_footer = \
- "Attached are the patch, license diff (if change) and bitbake log.\n" \
+ "Please review the attached files for further information and build/update failures.\n" \
"Any problem please file a bug at https://bugzilla.yoctoproject.org/enter_bug.cgi?product=Automated%20Update%20Handler\n\n" \
"Regards,\nThe Upgrade Helper"
@@ -333,20 +334,30 @@ class Updater(object):
if "status_recipients" in settings:
cc_addr = settings["status_recipients"].split()
- subject = "[AUH] " + pkg_ctx['PN'] + ": upgrading to " + pkg_ctx['NPV']
+ newversion = pkg_ctx['NPV'] if not pkg_ctx['NPV'].endswith("new-commits-available") else pkg_ctx['NSRCREV']
+ subject = "[AUH] " + pkg_ctx['PN'] + ": upgrading to " + newversion
if not pkg_ctx['error']:
subject += " SUCCEEDED"
else:
subject += " FAILED"
- msg_body = mail_header % (pkg_ctx['PN'], pkg_ctx['NPV'],
+ msg_body = mail_header % (pkg_ctx['PN'], newversion,
self._get_status_msg(pkg_ctx['error']))
- if 'recipe' in pkg_ctx:
- license_diff_fn = pkg_ctx['recipe'].get_license_diff_file_name()
- if license_diff_fn:
- msg_body += license_change_info % license_diff_fn
- if not pkg_ctx['error']:
- msg_body += next_steps_info % (', '.join(self.opts['machines']),
- os.path.basename(pkg_ctx['patch_file']))
+
+ if pkg_ctx['error'] is not None:
+ msg_body += """Detailed error information:
+
+%s
+%s
+%s
+
+""" %(pkg_ctx['error'].message if pkg_ctx['error'].message else "", pkg_ctx['error'].stdout if pkg_ctx['error'].stdout else "" , pkg_ctx['error'].stderr if pkg_ctx['error'].stderr else "")
+
+ if 'license_diff_fn' in pkg_ctx:
+ license_diff_fn = pkg_ctx['license_diff_fn']
+ msg_body += license_change_info % license_diff_fn
+
+ if 'patch_file' in pkg_ctx and pkg_ctx['patch_file'] != None:
+ msg_body += next_steps_info % (os.path.basename(pkg_ctx['patch_file']))
if self.opts['testimage']:
if 'integration_error' in pkg_ctx:
--
2.15.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [auh][PATCH 16/20] upgradehelper.py: send email even if recipe upgrade failed
2017-12-14 16:35 [auh][PATCH 00/20] Porting AUH to devtool and making it suitable for local use Alexander Kanavin
` (14 preceding siblings ...)
2017-12-14 16:35 ` [auh][PATCH 15/20] upgradehelper.py: fixups to the email message template Alexander Kanavin
@ 2017-12-14 16:35 ` Alexander Kanavin
2017-12-14 16:35 ` [auh][PATCH 17/20] upgradehelper.py: add all changes before committing them Alexander Kanavin
` (3 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Alexander Kanavin @ 2017-12-14 16:35 UTC (permalink / raw)
To: yocto
From: Alexander Kanavin <alex.kanavin@gmail.com>
Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
---
upgradehelper.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/upgradehelper.py b/upgradehelper.py
index d3cc2c4..043d334 100755
--- a/upgradehelper.py
+++ b/upgradehelper.py
@@ -380,8 +380,7 @@ class Updater(object):
if os.path.isfile(attachment_fullpath):
attachments.append(attachment_fullpath)
- # Only send email to Maintainer when recipe upgrade succeed.
- if self.opts['send_email'] and not pkg_ctx['error']:
+ if self.opts['send_email']:
self.email_handler.send_email(to_addr, subject, msg_body, attachments, cc_addr=cc_addr)
# Preserve email for review purposes.
email_file = os.path.join(pkg_ctx['workdir'],
--
2.15.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [auh][PATCH 17/20] upgradehelper.py: add all changes before committing them
2017-12-14 16:35 [auh][PATCH 00/20] Porting AUH to devtool and making it suitable for local use Alexander Kanavin
` (15 preceding siblings ...)
2017-12-14 16:35 ` [auh][PATCH 16/20] upgradehelper.py: send email even if recipe upgrade failed Alexander Kanavin
@ 2017-12-14 16:35 ` Alexander Kanavin
2017-12-14 16:35 ` [auh][PATCH 18/20] upgradehelper.py: when attempting to commit changes do not discard previous errors Alexander Kanavin
` (2 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Alexander Kanavin @ 2017-12-14 16:35 UTC (permalink / raw)
To: yocto
From: Alexander Kanavin <alex.kanavin@gmail.com>
Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
---
modules/utils/git.py | 3 +++
upgradehelper.py | 1 +
2 files changed, 4 insertions(+)
diff --git a/modules/utils/git.py b/modules/utils/git.py
index a6f5422..0e812b9 100644
--- a/modules/utils/git.py
+++ b/modules/utils/git.py
@@ -52,6 +52,9 @@ class Git(object):
def stash(self):
return self._cmd("stash")
+ def add(self, src):
+ return self._cmd("add " + src)
+
def commit(self, commit_message, author=None):
if author is None:
return self._cmd("commit -a -s -m \"" + commit_message + "\"")
diff --git a/upgradehelper.py b/upgradehelper.py
index 043d334..d1f441c 100755
--- a/upgradehelper.py
+++ b/upgradehelper.py
@@ -402,6 +402,7 @@ class Updater(object):
pkg_ctx['patch_file'] = None
I(" %s: Auto commit changes ..." % pkg_ctx['PN'])
+ self.git.add(pkg_ctx['recipe_dir'])
self.git.commit(pkg_ctx['commit_msg'], self.opts['author'])
stdout = self.git.create_patch(pkg_ctx['workdir'])
--
2.15.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [auh][PATCH 18/20] upgradehelper.py: when attempting to commit changes do not discard previous errors
2017-12-14 16:35 [auh][PATCH 00/20] Porting AUH to devtool and making it suitable for local use Alexander Kanavin
` (16 preceding siblings ...)
2017-12-14 16:35 ` [auh][PATCH 17/20] upgradehelper.py: add all changes before committing them Alexander Kanavin
@ 2017-12-14 16:35 ` Alexander Kanavin
2017-12-14 16:35 ` [auh][PATCH 19/20] upgradehelper.py: when recovering from upgrade error, do not refer to a dict entry that may not exist Alexander Kanavin
2017-12-14 16:36 ` [auh][PATCH 20/20] upgradehelper.py: revert commits that failed to build Alexander Kanavin
19 siblings, 0 replies; 21+ messages in thread
From: Alexander Kanavin @ 2017-12-14 16:35 UTC (permalink / raw)
To: yocto
From: Alexander Kanavin <alex.kanavin@gmail.com>
Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
---
upgradehelper.py | 11 ++---------
1 file changed, 2 insertions(+), 9 deletions(-)
diff --git a/upgradehelper.py b/upgradehelper.py
index d1f441c..3c41012 100755
--- a/upgradehelper.py
+++ b/upgradehelper.py
@@ -397,7 +397,6 @@ class Updater(object):
f.write("\n%s\n" % msg_body)
def commit_changes(self, pkg_ctx):
- fail = False
try:
pkg_ctx['patch_file'] = None
@@ -411,8 +410,7 @@ class Updater(object):
if not pkg_ctx['patch_file']:
msg = "Patch file not generated."
E(" %s: %s\n %s" % (pkg_ctx['PN'], msg, stdout))
- pkg_ctx['error'] = Error(msg, stdout)
- fail = True
+ raise Error(msg, stdout)
else:
I(" %s: Save patch in directory: %s." %
(pkg_ctx['PN'], pkg_ctx['workdir']))
@@ -425,12 +423,7 @@ class Updater(object):
I(" %s: %s" % (pkg_ctx['PN'], msg))
I(" %s: %s" % (pkg_ctx['PN'], e.stdout))
-
- pkg_ctx['error'] = Error(msg, e.stdout)
- fail = True
-
- if fail:
- raise pkg_ctx['error']
+ raise e
def send_status_mail(self, statistics_summary):
if "status_recipients" not in settings:
--
2.15.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [auh][PATCH 19/20] upgradehelper.py: when recovering from upgrade error, do not refer to a dict entry that may not exist
2017-12-14 16:35 [auh][PATCH 00/20] Porting AUH to devtool and making it suitable for local use Alexander Kanavin
` (17 preceding siblings ...)
2017-12-14 16:35 ` [auh][PATCH 18/20] upgradehelper.py: when attempting to commit changes do not discard previous errors Alexander Kanavin
@ 2017-12-14 16:35 ` Alexander Kanavin
2017-12-14 16:36 ` [auh][PATCH 20/20] upgradehelper.py: revert commits that failed to build Alexander Kanavin
19 siblings, 0 replies; 21+ messages in thread
From: Alexander Kanavin @ 2017-12-14 16:35 UTC (permalink / raw)
To: yocto
From: Alexander Kanavin <alex.kanavin@gmail.com>
Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
---
upgradehelper.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/upgradehelper.py b/upgradehelper.py
index 3c41012..58fa0a6 100755
--- a/upgradehelper.py
+++ b/upgradehelper.py
@@ -510,7 +510,7 @@ class Updater(object):
E(" %s: %s" % (pkg_ctx['PN'], e.message))
- if os.listdir(pkg_ctx['workdir']):
+ if 'workdir' in pkg_ctx and os.listdir(pkg_ctx['workdir']):
E(" %s: Upgrade FAILED! Logs and/or file diffs are available in %s"
% (pkg_ctx['PN'], pkg_ctx['workdir']))
--
2.15.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [auh][PATCH 20/20] upgradehelper.py: revert commits that failed to build
2017-12-14 16:35 [auh][PATCH 00/20] Porting AUH to devtool and making it suitable for local use Alexander Kanavin
` (18 preceding siblings ...)
2017-12-14 16:35 ` [auh][PATCH 19/20] upgradehelper.py: when recovering from upgrade error, do not refer to a dict entry that may not exist Alexander Kanavin
@ 2017-12-14 16:36 ` Alexander Kanavin
19 siblings, 0 replies; 21+ messages in thread
From: Alexander Kanavin @ 2017-12-14 16:36 UTC (permalink / raw)
To: yocto
From: Alexander Kanavin <alex.kanavin@gmail.com>
This helps avoid 'cascading build failures' where one failed update
holds up everything else.
Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
---
modules/utils/git.py | 3 +++
upgradehelper.py | 3 +++
2 files changed, 6 insertions(+)
diff --git a/modules/utils/git.py b/modules/utils/git.py
index 0e812b9..749d24d 100644
--- a/modules/utils/git.py
+++ b/modules/utils/git.py
@@ -61,6 +61,9 @@ class Git(object):
else:
return self._cmd("commit -a --author=\"" + author + "\" -m \"" + commit_message + "\"")
+ def revert(self, commit):
+ return self._cmd("revert --no-edit " + commit)
+
def create_patch(self, out_dir):
return self._cmd("format-patch -M10 -1 -o " + out_dir)
diff --git a/upgradehelper.py b/upgradehelper.py
index 58fa0a6..71ee0b0 100755
--- a/upgradehelper.py
+++ b/upgradehelper.py
@@ -414,6 +414,9 @@ class Updater(object):
else:
I(" %s: Save patch in directory: %s." %
(pkg_ctx['PN'], pkg_ctx['workdir']))
+ if pkg_ctx['error'] is not None:
+ I("Due to build errors, the commit will also be reverted to avoid cascading upgrade failures.")
+ self.git.revert("HEAD")
except Error as e:
msg = ''
--
2.15.0
^ permalink raw reply related [flat|nested] 21+ messages in thread