* [PATCH 1/4] recipeutils: Add get_recipe_upstream_version and get_recipe_pv_without_srcpv functions
2015-06-02 19:49 [PATCH 0/4] recipeutils adds get_recipe_upstream_version and distrodata fixes Aníbal Limón
@ 2015-06-02 19:49 ` Aníbal Limón
2015-06-02 19:49 ` [PATCH 2/4] distrodata: Remove unnecessary include of package_regex.inc Aníbal Limón
` (3 subsequent siblings)
4 siblings, 0 replies; 9+ messages in thread
From: Aníbal Limón @ 2015-06-02 19:49 UTC (permalink / raw)
To: openembedded-core
The get_recipe_upstream_version functions tries to get the current
version of recipe in upstream it uses bb.fetch2 latest_versionstring
method also latest_revision when is SCM.
The get_recipe_pv_without_srcpv discards the SRCPV in SCM's recipe like
git it returns a tuple with the version, prefix and suffix of a PV.
Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
---
meta/lib/oe/recipeutils.py | 88 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 88 insertions(+)
diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py
index f05b6c0..26bbf3e 100644
--- a/meta/lib/oe/recipeutils.py
+++ b/meta/lib/oe/recipeutils.py
@@ -626,3 +626,91 @@ def replace_dir_vars(path, d):
path = path.replace(dirpath, '${%s}' % dirvars[dirpath])
return path
+def get_recipe_pv_without_srcpv(pv, uri_type):
+ """
+ Get PV without SRCPV common in SCM's for now only
+ support git.
+
+ Returns tuple with pv, prefix and suffix.
+ """
+ pfx = ''
+ sfx = ''
+
+ if uri_type == 'git':
+ git_regex = re.compile("(?P<pfx>(v|))(?P<ver>((\d+[\.\-_]*)+))(?P<sfx>(\+|)(git|)(r|)(AUTOINC|)(\+|))(?P<rev>.*)")
+ m = git_regex.match(pv)
+
+ if m:
+ pv = m.group('ver')
+ pfx = m.group('pfx')
+ sfx = m.group('sfx')
+
+ return (pv, pfx, sfx)
+
+def get_recipe_upstream_version(rd):
+ """
+ Get upstream version of recipe using bb.fetch2 methods with support for
+ http, https, ftp and git.
+
+ bb.fetch2 exceptions can be raised,
+ FetchError when don't have network access or upstream site don't response.
+ NoMethodError when uri latest_versionstring method isn't implemented.
+
+ Returns a dictonary with version, type and datetime.
+ Type can be A for Automatic, M for Manual and U for Unknown.
+ """
+ from bb.fetch2 import decodeurl
+ from datetime import datetime
+
+ ru = {}
+ ru['version'] = ''
+ ru['type'] = 'U'
+ ru['datetime'] = ''
+
+ # XXX: we suppose that the first entry points to the upstream sources
+ src_uri = rd.getVar('SRC_URI', True).split()[0]
+ uri_type, _, _, _, _, _ = decodeurl(src_uri)
+
+ pv = rd.getVar('PV', True)
+
+ manual_upstream_version = rd.getVar("RECIPE_UPSTREAM_VERSION", True)
+ if manual_upstream_version:
+ # manual tracking of upstream version.
+ ru['version'] = manual_upstream_version
+ ru['type'] = 'M'
+
+ manual_upstream_date = rd.getVar("CHECK_DATE", True)
+ if manual_upstream_date:
+ date = datetime.strptime(manual_upstream_date, "%b %d, %Y")
+ else:
+ date = datetime.now()
+ ru['datetime'] = date
+
+ elif uri_type == "file":
+ # files are always up-to-date
+ ru['version'] = pv
+ ru['type'] = 'A'
+ ru['datetime'] = datetime.now()
+ else:
+ ud = bb.fetch2.FetchData(src_uri, rd)
+ pupver = ud.method.latest_versionstring(ud, rd)
+
+ if uri_type == 'git':
+ (pv, pfx, sfx) = get_recipe_pv_without_srcpv(pv, uri_type)
+
+ latest_revision = ud.method.latest_revision(ud, rd, ud.names[0])
+
+ # if contains revision but not pupver use current pv
+ if pupver == '' and latest_revision:
+ pupver = pv
+
+ if pupver != '':
+ pupver = pfx + pupver + sfx + latest_revision[:10]
+
+ if pupver != '':
+ ru['version'] = pupver
+ ru['type'] = 'A'
+
+ ru['datetime'] = datetime.now()
+
+ return ru
--
1.9.1
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH 3/4] distrodata: checkpkg make usage of oe.recipeutils.get_recipe_upstream_version
2015-06-02 19:49 [PATCH 0/4] recipeutils adds get_recipe_upstream_version and distrodata fixes Aníbal Limón
2015-06-02 19:49 ` [PATCH 1/4] recipeutils: Add get_recipe_upstream_version and get_recipe_pv_without_srcpv functions Aníbal Limón
2015-06-02 19:49 ` [PATCH 2/4] distrodata: Remove unnecessary include of package_regex.inc Aníbal Limón
@ 2015-06-02 19:49 ` Aníbal Limón
2015-06-02 19:49 ` [PATCH 4/4] distrodata: Use Python CSV instead of did by hand Aníbal Limón
2015-06-04 16:09 ` [PATCH 0/4] recipeutils adds get_recipe_upstream_version and distrodata fixes Burton, Ross
4 siblings, 0 replies; 9+ messages in thread
From: Aníbal Limón @ 2015-06-02 19:49 UTC (permalink / raw)
To: openembedded-core
Now get_recipe_upstream_version function exists in oe.recipeutils module
to avoid duplicate code make usage of it.
Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
---
meta/classes/distrodata.bbclass | 84 ++++++++++++++++++-----------------------
1 file changed, 37 insertions(+), 47 deletions(-)
diff --git a/meta/classes/distrodata.bbclass b/meta/classes/distrodata.bbclass
index e1fc6dd..092c372 100644
--- a/meta/classes/distrodata.bbclass
+++ b/meta/classes/distrodata.bbclass
@@ -266,11 +266,15 @@ python do_checkpkg() {
import re
import tempfile
import subprocess
+ import oe.recipeutils
+ from bb.utils import vercmp_string
+ from bb.fetch2 import FetchError, NoMethodError, decodeurl
"""first check whether a uri is provided"""
src_uri = d.getVar('SRC_URI', True)
if not src_uri:
return
+ uri_type, _, _, _, _, _ = decodeurl(src_uri)
"""initialize log files."""
logpath = d.getVar('LOG_DIR', True)
@@ -310,10 +314,7 @@ python do_checkpkg() {
pdesc = localdata.getVar('DESCRIPTION', True)
pgrp = localdata.getVar('SECTION', True)
- if localdata.getVar('PRSPV', True):
- pversion = localdata.getVar('PRSPV', True)
- else:
- pversion = localdata.getVar('PV', True)
+ pversion = localdata.getVar('PV', True)
plicense = localdata.getVar('LICENSE', True)
psection = localdata.getVar('SECTION', True)
phome = localdata.getVar('HOMEPAGE', True)
@@ -325,61 +326,50 @@ python do_checkpkg() {
maintainer = localdata.getVar('RECIPE_MAINTAINER', True)
""" Get upstream version version """
- pupver = None
- pstatus = "ErrUnknown"
- found = 0
-
- for uri in src_uri.split():
- m = re.compile('(?P<type>[^:]*)').match(uri)
- if not m:
- raise MalformedUrl(uri)
- elif m.group('type') in ('http', 'https', 'ftp', 'cvs', 'svn', 'git'):
- found = 1
- psrcuri = uri
- pproto = m.group('type')
- break
- if not found:
- pproto = "file"
-
- if pproto in ['http', 'https', 'ftp', 'git']:
- try:
- ud = bb.fetch2.FetchData(psrcuri, d)
- pupver = ud.method.latest_versionstring(ud, d)
- if pproto == 'git':
- if pupver == "":
- pupver = pversion.rsplit("+")[0]
- if re.search(pversion, "gitrAUTOINC"):
- pupver += "+gitrAUTOINC+"
- else:
- pupver += "+gitAUTOINC+"
- latest_revision = ud.method.latest_revision(ud, d, ud.names[0])
- pupver += latest_revision[:10]
- except Exception as inst:
- bb.warn("%s: unexpected error: %s" % (pname, repr(inst)))
+ pupver = ""
+ pstatus = ""
+
+ try:
+ uv = oe.recipeutils.get_recipe_upstream_version(localdata)
+
+ pupver = uv['version']
+ except Exception as e:
+ if e is FetchError:
pstatus = "ErrAccess"
- elif pproto == "file":
- """Local files are always updated"""
- pupver = pversion
- else:
- pstatus = "ErrUnsupportedProto"
- bb.note("do_checkpkg, protocol %s isn't implemented" % pproto)
+ elif e is NoMethodError:
+ pstatus = "ErrUnsupportedProto"
+ else:
+ pstatus = "ErrUnknown"
+ """Set upstream version status"""
if not pupver:
pupver = "N/A"
- elif pupver == pversion:
- pstatus = "MATCH"
else:
- pstatus = "UPDATE"
+ pv, _, _ = oe.recipeutils.get_recipe_pv_without_srcpv(pversion, uri_type)
+ upv, _, _ = oe.recipeutils.get_recipe_pv_without_srcpv(pupver, uri_type)
+
+ cmp = vercmp_string(pv, upv)
+ if cmp == -1:
+ pstatus = "UPDATE"
+ elif cmp == 0:
+ pstatus = "MATCH"
"""Read from manual distro tracking fields as alternative"""
pmver = d.getVar("RECIPE_UPSTREAM_VERSION", True)
if not pmver:
pmver = "N/A"
pmstatus = "ErrNoRecipeData"
- elif pmver == pupver:
- pmstatus = "MATCH"
else:
- pmstatus = "UPDATE"
+ mpv, _, _ = oe.recipeutils.get_recipe_pv_without_srcpv(pmver, uri_type)
+ upv, _, _ = oe.recipeutils.get_recipe_pv_without_srcpv(pupver, uri_type)
+
+ cmp = vercmp_string(mpv, upv)
+ if cmp == -1:
+ pmstatus = "UPDATE"
+ elif cmp == 0:
+ pmstatus = "MATCH"
+ else:
+ pmstatus = ""
pdepends = "".join(pdepends.split("\t"))
pdesc = "".join(pdesc.split("\t"))
--
1.9.1
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH 4/4] distrodata: Use Python CSV instead of did by hand
2015-06-02 19:49 [PATCH 0/4] recipeutils adds get_recipe_upstream_version and distrodata fixes Aníbal Limón
` (2 preceding siblings ...)
2015-06-02 19:49 ` [PATCH 3/4] distrodata: checkpkg make usage of oe.recipeutils.get_recipe_upstream_version Aníbal Limón
@ 2015-06-02 19:49 ` Aníbal Limón
2015-06-04 16:09 ` [PATCH 0/4] recipeutils adds get_recipe_upstream_version and distrodata fixes Burton, Ross
4 siblings, 0 replies; 9+ messages in thread
From: Aníbal Limón @ 2015-06-02 19:49 UTC (permalink / raw)
To: openembedded-core
Fix CSV generation in distrodata class using Python CSV
module before it some errors happen when read due to
incorrect quoting/delimiters.
[YOCTO #7777]
Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
---
meta/classes/distrodata.bbclass | 125 ++++++++++++++++++++--------------------
1 file changed, 64 insertions(+), 61 deletions(-)
diff --git a/meta/classes/distrodata.bbclass b/meta/classes/distrodata.bbclass
index 092c372..0cefa7a 100644
--- a/meta/classes/distrodata.bbclass
+++ b/meta/classes/distrodata.bbclass
@@ -2,11 +2,16 @@ addhandler distro_eventhandler
distro_eventhandler[eventmask] = "bb.event.BuildStarted"
python distro_eventhandler() {
import oe.distro_check as dc
+ import csv
logfile = dc.create_log_file(e.data, "distrodata.csv")
+
lf = bb.utils.lockfile("%s.lock" % logfile)
- f = open(logfile, "a")
- f.write("Package,Description,Owner,License,VerMatch,Version,Upsteam,Reason,Recipe Status,Distro 1,Distro 2,Distro 3\n")
- f.close()
+ with open(logfile, "a") as f:
+ writer = csv.writer(f)
+ writer.writerow(['Package', 'Description', 'Owner', 'License',
+ 'VerMatch', 'Version', 'Upsteam', 'Reason', 'Recipe Status',
+ 'Distro 1', 'Distro 2', 'Distro 3'])
+ f.close()
bb.utils.unlockfile(lf)
return
@@ -98,6 +103,7 @@ python do_distrodata_np() {
addtask distrodata
do_distrodata[nostamp] = "1"
python do_distrodata() {
+ import csv
logpath = d.getVar('LOG_DIR', True)
bb.utils.mkdirhier(logpath)
logfile = os.path.join(logpath, "distrodata.csv")
@@ -176,14 +182,13 @@ python do_distrodata() {
result = dist_check.compare_in_distro_packages_list(distro_check_dir, localdata)
lf = bb.utils.lockfile("%s.lock" % logfile)
- f = open(logfile, "a")
- f.write("%s,%s,%s,%s,%s,%s,%s,%s,%s" % \
- (pname, pdesc, maintainer, plicense, vermatch, pcurver, pupver, noupdate_reason, rstatus))
- line = ""
- for i in result:
- line = line + "," + i
- f.write(line + "\n")
- f.close()
+ with open(logfile, "a") as f:
+ row = [pname, pdesc, maintainer, plicense, vermatch, pcurver, pupver, noupdate_reason, rstatus]
+ row.extend(result)
+
+ writer = csv.writer(f)
+ writer.writerow(row)
+ f.close()
bb.utils.unlockfile(lf)
}
@@ -198,45 +203,33 @@ do_distrodataall() {
addhandler checkpkg_eventhandler
checkpkg_eventhandler[eventmask] = "bb.event.BuildStarted bb.event.BuildCompleted"
python checkpkg_eventhandler() {
+ import csv
+
def parse_csv_file(filename):
package_dict = {}
- fd = open(filename, "r")
- lines = fd.read().rsplit("\n")
- fd.close()
-
- first_line = ''
- index = 0
- for line in lines:
- #Skip the first line
- if index == 0:
- first_line = line
- index += 1
- continue
- elif line == '':
- continue
- index += 1
- package_name = line.rsplit("\t")[0]
- if '-native' in package_name or 'nativesdk-' in package_name:
- original_name = package_name.rsplit('-native')[0]
- if original_name == '':
- original_name = package_name.rsplit('nativesdk-')[0]
- if original_name in package_dict:
+
+ with open(filename, "r") as f:
+ reader = csv.reader(f, delimiter='\t')
+ for row in reader:
+ pn = row[0]
+
+ if reader.line_num == 1:
+ header = row
continue
- else:
- package_dict[package_name] = line
- else:
- new_name = package_name + "-native"
- if not(new_name in package_dict):
- new_name = 'nativesdk-' + package_name
- if new_name in package_dict:
- del package_dict[new_name]
- package_dict[package_name] = line
-
- fd = open(filename, "w")
- fd.write("%s\n"%first_line)
- for el in package_dict:
- fd.write(package_dict[el] + "\n")
- fd.close()
+
+ if '-native' in pn or 'nativesdk-' in pn:
+ continue
+
+ if not pn in package_dict.keys():
+ package_dict[pn] = row
+ f.close()
+
+ with open(filename, "w") as f:
+ writer = csv.writer(f, delimiter='\t')
+ writer.writerow(header)
+ for pn in package_dict.keys():
+ writer.writerow(package_dict[pn])
+ f.close()
del package_dict
@@ -245,9 +238,13 @@ python checkpkg_eventhandler() {
logfile = dc.create_log_file(e.data, "checkpkg.csv")
lf = bb.utils.lockfile("%s.lock" % logfile)
- f = open(logfile, "a")
- f.write("Package\tVersion\tUpver\tLicense\tSection\tHome\tRelease\tDepends\tBugTracker\tPE\tDescription\tStatus\tTracking\tURI\tMAINTAINER\tNoUpReason\n")
- f.close()
+ with open(logfile, "a") as f:
+ writer = csv.writer(f, delimiter='\t')
+ headers = ['Package', 'Version', 'Upver', 'License', 'Section',
+ 'Home', 'Release', 'Depends', 'BugTracker', 'PE', 'Description',
+ 'Status', 'Tracking', 'URI', 'MAINTAINER', 'NoUpReason']
+ writer.writerow(headers)
+ f.close()
bb.utils.unlockfile(lf)
elif bb.event.getName(e) == "BuildCompleted":
import os
@@ -263,6 +260,7 @@ addtask checkpkg
do_checkpkg[nostamp] = "1"
python do_checkpkg() {
localdata = bb.data.createCopy(d)
+ import csv
import re
import tempfile
import subprocess
@@ -375,10 +373,12 @@ python do_checkpkg() {
pdesc = "".join(pdesc.split("\t"))
no_upgr_reason = d.getVar('RECIPE_NO_UPDATE_REASON', True)
lf = bb.utils.lockfile("%s.lock" % logfile)
- f = open(logfile, "a")
- f.write("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" % \
- (pname,pversion,pupver,plicense,psection, phome,prelease, pdepends,pbugtracker,ppe,pdesc,pstatus,pmver,psrcuri,maintainer, no_upgr_reason))
- f.close()
+ with open(logfile, "a") as f:
+ writer = csv.writer(f, delimiter='\t')
+ writer.writerow([pname, pversion, pupver, plicense, psection, phome,
+ prelease, pdepends, pbugtracker, ppe, pdesc, pstatus, pmver,
+ psrcuri, maintainer, no_upgr_reason])
+ f.close()
bb.utils.unlockfile(lf)
}
@@ -441,12 +441,14 @@ addhandler checklicense_eventhandler
checklicense_eventhandler[eventmask] = "bb.event.BuildStarted"
python checklicense_eventhandler() {
"""initialize log files."""
+ import csv
import oe.distro_check as dc
logfile = dc.create_log_file(e.data, "missinglicense.csv")
lf = bb.utils.lockfile("%s.lock" % logfile)
- f = open(logfile, "a")
- f.write("Package\tLicense\tMissingLicense\n")
- f.close()
+ with open(logfile, "a") as f:
+ writer = csv.writer(f, delimiter='\t')
+ writer.writerow(['Package', 'License', 'MissingLicense'])
+ f.close()
bb.utils.unlockfile(lf)
return
}
@@ -454,6 +456,7 @@ python checklicense_eventhandler() {
addtask checklicense
do_checklicense[nostamp] = "1"
python do_checklicense() {
+ import csv
import shutil
logpath = d.getVar('LOG_DIR', True)
bb.utils.mkdirhier(logpath)
@@ -466,10 +469,10 @@ python do_checklicense() {
.replace(',', '').replace(" ", "").split("&"))):
if not os.path.isfile(os.path.join(generic_directory, license_type)):
lf = bb.utils.lockfile("%s.lock" % logfile)
- f = open(logfile, "a")
- f.write("%s\t%s\t%s\n" % \
- (pn,license_types,license_type))
- f.close()
+ with open(logfile, "a") as f:
+ writer = csv.writer(f, delimiter='\t')
+ writer.writerow([pn, license_types, license_type])
+ f.close()
bb.utils.unlockfile(lf)
return
}
--
1.9.1
^ permalink raw reply related [flat|nested] 9+ messages in thread