[PATCH 0/7] Realize archiving functions

[PATCH 0/7] Realize archiving functions

[Xiaofeng Yan]

From: Xiaofeng Yan <xiaofeng.yan@windriver.com>

archiver.bbclass is for archiving sources , patches and logs including .bb and .inc files by configuring conf/local.conf  which control what a archiving package should include.
For archiving a source rpm package, I add some functions to package_rpm.bbclass.
I also merge source_pkg.bbclass src_distribute.bbclass src_distribute_local.bbclass to archiver.bbclass due to their similar functions.

Pull URL: git://git.pokylinux.org/poky-contrib.git
  Branch: xiaofeng/1977
  Browse: http://git.pokylinux.org/cgit.cgi/poky-contrib/log/?h=xiaofeng/1977

Thanks,
    Xiaofeng Yan <xiaofeng.yan@windriver.com>
---


Xiaofeng Yan (7):
  package_rpm: Add srpm function to this bbclass
  archiver.bbclass: New bbclass for archiving sources, patches,logs and
    scripts
  archive-patched-source.bbclass: Archive patched source
  archive-configured-source.bbclass: Archive configured source
  archive-original-source.bbclass: Archive original source
  local.conf.sample: Add set for archiving packages
  sourcepkg, src_distribute, src_distribute_local: remove three bbclass

 meta-yocto/conf/local.conf.sample              |   25 ++
 meta/classes/archive-configured-source.bbclass |   14 +
 meta/classes/archive-original-source.bbclass   |   14 +
 meta/classes/archive-patched-source.bbclass    |   14 +
 meta/classes/archiver.bbclass                  |  421 ++++++++++++++++++++++++
 meta/classes/package_rpm.bbclass               |   63 ++++
 meta/classes/sourcepkg.bbclass                 |  107 ------
 meta/classes/src_distribute.bbclass            |   49 ---
 meta/classes/src_distribute_local.bbclass      |   33 --
 9 files changed, 551 insertions(+), 189 deletions(-)
 create mode 100644 meta/classes/archive-configured-source.bbclass
 create mode 100644 meta/classes/archive-original-source.bbclass
 create mode 100644 meta/classes/archive-patched-source.bbclass
 create mode 100644 meta/classes/archiver.bbclass
 delete mode 100644 meta/classes/sourcepkg.bbclass
 delete mode 100644 meta/classes/src_distribute.bbclass
 delete mode 100644 meta/classes/src_distribute_local.bbclass

[PATCH 1/7] package_rpm: Add srpm function to this bbclass

[Xiaofeng Yan]

From: Xiaofeng Yan <xiaofeng.yan@windriver.com>

Add a new function to archive source, patches and logs to a source rpm
package. Every source rpm package will be deployed to
${DEPLOY_DIR}/sources/deploy-srpm.

[YOCTO #1977]

Signed-off-by: Xiaofeng Yan <xiaofeng.yan@windriver.com>
---
 meta/classes/package_rpm.bbclass |   63 ++++++++++++++++++++++++++++++++++++++
 1 files changed, 63 insertions(+), 0 deletions(-)

diff --git a/meta/classes/package_rpm.bbclass b/meta/classes/package_rpm.bbclass
index 68313ec..b261a0e 100644
--- a/meta/classes/package_rpm.bbclass
+++ b/meta/classes/package_rpm.bbclass
@@ -6,6 +6,7 @@ RPM="rpm"
 RPMBUILD="rpmbuild"
 
 PKGWRITEDIRRPM = "${WORKDIR}/deploy-rpms"
+PKGWRITEDIRSRPM = "${DEPLOY_DIR}/sources/deploy-srpm"
 
 python package_rpm_fn () {
 	d.setVar('PKGFN', d.getVar('PKG'))
@@ -475,6 +476,37 @@ python write_specfile () {
 	import textwrap
 	import oe.packagedata
 
+	# append information for logs and patches to %prep
+	def add_prep(d,spec_files_bottom):
+		if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True) and d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() == 'SRPM':
+			spec_files_bottom.append('%%prep -n %s' % d.getVar('PN', True) )
+			spec_files_bottom.append('%s' % "echo \"include logs and patches, Please check them in SOURCES\"")
+			spec_files_bottom.append('')
+
+	# get the name of tarball for sources, patches and logs
+	def get_tarballs(d):
+		if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True) and d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() == 'SRPM':
+			sourcelist=[]
+			workdir = d.getVar('WORKDIR',True)
+			print os.listdir(workdir)
+			for source in os.listdir(workdir):
+				if 'tar.gz' in source:
+					sourcelist.append(source)
+			return sourcelist
+
+	# append the name of tarball to key word 'SOURCE' in xxx.spec.
+	def tail_source(d,source_list=[],patch_list=None):
+		if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True) and d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() == 'SRPM':
+			source_number = 0
+			patch_number = 0
+			for source in source_list:
+				spec_preamble_top.append('Source' + str(source_number) + ': %s' % source)
+				source_number += 1
+			if patch_list:
+				for patch in patch_list:
+					print_deps(patch, "Patch" + str(patch_number), spec_preamble_top, d)
+					patch_number += 1
+
 	# We need a simple way to remove the MLPREFIX from the package name,
 	# and dependency information...
 	def strip_multilib(name, d):
@@ -707,6 +739,8 @@ python write_specfile () {
 			spec_preamble_bottom.append('License: %s' % splitlicense)
 		spec_preamble_bottom.append('Group: %s' % splitsection)
 
+		source_list = get_tarballs(d)
+		tail_source(d,source_list,None)
 		# Replaces == Obsoletes && Provides
 		if splitrreplaces and splitrreplaces.strip() != "":
 			for dep in splitrreplaces.split(','):
@@ -786,6 +820,7 @@ python write_specfile () {
 		del localdata
 		bb.utils.unlockfile(lf)
 
+	add_prep(d,spec_files_bottom)
 	spec_preamble_top.append('Summary: %s' % srcsummary)
 	spec_preamble_top.append('Name: %s' % srcname)
 	spec_preamble_top.append('Version: %s' % srcversion)
@@ -900,6 +935,25 @@ python write_specfile () {
 python do_package_rpm () {
 	import os
 
+	def creat_srpm_dir(d):
+		if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True) and d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() == 'SRPM':
+			licenses = d.getVar('LICENSE', 1).replace('&', '|')
+			licenses = licenses.replace('(', '').replace(')', '')
+			clean_licenses = ""
+			for x in licenses.split():
+				if x.strip() == '' or x == 'CLOSED':
+					continue
+				if x != "|":
+					clean_licenses += x
+				if '|' in clean_licenses:
+					clean_licenses = clean_licenses.replace('|','')
+
+			pkgwritesrpmdir = bb.data.expand('${PKGWRITEDIRSRPM}/${PACKAGE_ARCH_EXTEND}', d)
+			pkgwritesrpmdir = pkgwritesrpmdir + '/' + clean_licenses
+			bb.mkdirhier(pkgwritesrpmdir)
+			os.chmod(pkgwritesrpmdir, 0755)
+			return pkgwritesrpmdir
+
 	# We need a simple way to remove the MLPREFIX from the package name,
 	# and dependency information...
 	def strip_multilib(name, d):
@@ -1015,8 +1069,17 @@ python do_package_rpm () {
 	cmd = cmd + " --define 'debug_package %{nil}'"
 	cmd = cmd + " --define '_rpmfc_magic_path " + magicfile + "'"
 	cmd = cmd + " --define '_tmppath " + workdir + "'"
+	if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True) and d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() == 'SRPM':
+		cmdsrpm = cmd + " --define '_sourcedir " + workdir + "' --define '_srcrpmdir " + creat_srpm_dir(d) + "'"
+		cmdsrpm = 'fakeroot ' + cmdsrpm + " -bs " + outspecfile
 	cmd = cmd + " -bb " + outspecfile
 
+	# Build the source rpm package !
+	if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True) and d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() == 'SRPM':
+		d.setVar('SBUILDSPEC', cmdsrpm + "\n")
+		d.setVarFlag('SBUILDSPEC', 'func', '1')
+		bb.build.exec_func('SBUILDSPEC', d)
+
 	# Build the rpm package!
 	d.setVar('BUILDSPEC', cmd + "\n")
 	d.setVarFlag('BUILDSPEC', 'func', '1')
-- 
1.7.0.4

[PATCH 2/7] archiver.bbclass: New bbclass for archiving sources, patches, logs and scripts

[Xiaofeng Yan]

From: Xiaofeng Yan <xiaofeng.yan@windriver.com>

1 Archive sources in ${S} in the different stage (do_unpack,do_patch,do_configure).
2 Archive patches including series
3 Archive logs including scripts (.bb and .inc files)
4 dump environment resources which show all variable and
  functions used to xxx.showdata.dump when running a task
5 dump all content in 's' including patches to file xxx.diff.gz

All archiving packages  will be deployed to ${DEPLOY_DIR}/sources/

[#YOCTO 1977]

Signed-off-by: Xiaofeng Yan <xiaofeng.yan@windriver.com>
---
 meta/classes/archiver.bbclass |  421 +++++++++++++++++++++++++++++++++++++++++
 1 files changed, 421 insertions(+), 0 deletions(-)
 create mode 100644 meta/classes/archiver.bbclass

diff --git a/meta/classes/archiver.bbclass b/meta/classes/archiver.bbclass
new file mode 100644
index 0000000..95e870d
--- /dev/null
+++ b/meta/classes/archiver.bbclass
@@ -0,0 +1,421 @@
+# This file is used for archiving sources ,patches,and logs to tarball.
+# It also output building environment to xxx.dump.data and create xxx.diff.gz to record 
+# all content in ${S} to a diff file. 
+
+EXCLUDE_FROM ?= ".pc autom4te.cache"
+ARCHIVE_TYPE ?= "TAR SRPM"
+DISTRO ?= "poky"
+
+def parse_var(d,var):
+	''' parse variable like ${PV}  in "require xxx_${PV}.inc"  to a real value. for example, change "require xxx_${PV}.inc" to "require xxx_1.2.inc" '''
+	import re 
+	pat = re.compile('.*\$({(.*)}).*')
+	if '$' not in var and '/' not in var:
+		return var
+	else:
+		if '/' in var:
+			return [i for i in var.split('/') if i.endswith('.inc')][0]
+		elif '$' in  var:
+			m = pat.match(var)
+			patstr = '\$' + m.group(1)
+			var_str = m.group(2)
+			return re.sub(patstr,d.getVar(var_str,True),var)
+		else:
+			return var
+		
+
+def get_bb_inc(d):
+	'''create a directory "script-logs" including .bb and .inc file in ${WORKDIR}'''
+	import re
+	import os
+	import shutil
+	
+	bbinc = []
+	pat=re.compile('require\s*([^\s]*\.*)(.*)')
+	file_dir = d.getVar('FILE', True)
+	bbdir = os.path.dirname(file_dir)
+	work_dir = d.getVar('WORKDIR', True)
+	os.chdir(work_dir)
+	bb.mkdirhier("script-logs")
+	os.chdir(bbdir)
+	bbfile = os.path.basename(file_dir)
+	bbinc.append(bbfile)
+	
+	def get_inc (file):
+		f = open(file,'r')
+		for line in f.readlines():
+			if 'require' not  in line:
+				bbinc.append(file)
+			else:
+				try:
+					incfile = pat.match(line).group(1)
+					incfile = parse_var(d,incfile)
+					bbinc.append(incfile)
+					get_inc(incfile)
+				except AttributeError:
+					pass
+	get_inc(bbfile)
+	os.chdir(work_dir)
+	for root, dirs, files in os.walk(bbdir):
+		for file in bbinc:
+			if file in files:
+				shutil.copy(root + '/' + file,'script-logs')
+	oe.path.copytree('temp', 'script-logs')
+	return work_dir + '/script-logs'
+
+def get_all_patches(d):
+	'''copy patches and series file to a pointed directory which will be archived to tarball in ${WORKDIR}'''
+	import shutil
+	
+	src_patches=[]
+	pf = d.getVar('PF', True)
+	work_dir = d.getVar('WORKDIR', True)
+	dest = os.path.join(work_dir, pf + '-patches') 
+	shutil.rmtree(dest, ignore_errors=True)
+	bb.mkdirhier(dest)
+	
+	src_uri = d.getVar('SRC_URI', 1).split()
+	fetch = bb.fetch2.Fetch(src_uri, d)
+	locals = (fetch.localpath(url) for url in fetch.urls)
+	for local in locals:
+		src_patches.append(local)
+	for patch in src_patches[1:]:
+		 shutil.copy(patch,dest)
+	return dest
+
+def get_applying_patches(d):
+	"""only copy applying patches to a pointed directory which will be archived to tarball"""
+	import os
+	import shutil
+
+
+	pf = d.getVar('PF', True)
+	work_dir = d.getVar('WORKDIR', True)
+	dest = os.path.join(work_dir, pf + '-patches') 
+	shutil.rmtree(dest, ignore_errors=True)
+	bb.mkdirhier(dest)
+
+
+	patches = src_patches(d)
+	for patch in patches:
+		_, _, local, _, _, parm = bb.decodeurl(patch)
+		if local:
+			 shutil.copy(local,dest)
+	return dest
+
+def not_tarball(d):
+	'''packages including key words 'work-shared','native', 'task-' will be passed'''
+	import os
+
+	workdir = d.getVar('WORKDIR',True)
+	s = d.getVar('S',True)
+	if 'work-shared' in s or 'task-' in workdir or 'native' in workdir:
+		return True
+	else:
+		return False
+
+def archive_sources(d,middle_name): 
+	'''archive sources codes tree to tarball'''
+	import tarfile
+	import shutil
+	
+	s = d.getVar('S',True)
+	workdir=d.getVar('WORKDIR', True)
+	PF = d.getVar('PF',True)
+	tarname = PF + '-' + middle_name + ".tar.gz"
+
+	if os.path.exists(s) and s is not workdir:
+		sourcedir = os.path.basename(s)
+		tarbase = os.path.dirname(s)
+		if not sourcedir or os.path.dirname(tarbase) == workdir:
+			sourcedir = os.path.basename(os.path.dirname(s))
+			tarbase = os.path.dirname(os.path.dirname(s))
+		os.chdir(tarbase)
+	else:
+		sourcedir = os.path.basename(s)
+		if not os.path.exists(sourcedir):
+			os.mkdir(sourcedir)
+		try:
+			for file in os.listdir(s):
+				if file is not 'temp' and file is not sourcedir:
+					shutil.copy(file,sourcedir)
+		except (IOError,OSError):
+			pass
+
+
+	if (len(os.listdir(sourcedir))) != 0:
+		tar = tarfile.open( tarname, "w:gz")
+		tar.add(sourcedir)
+		tar.close()
+		if cmp(workdir,os.path.dirname(s)) and not os.path.exists(workdir + '/' + tarname):
+			shutil.move(os.path.dirname(s) + '/' + tarname,workdir)
+	else:
+		return
+	return tarname
+
+def archive_patches(d,patchdir,series):
+	'''archive patches to tarball and also include series files if 'series' is True'''
+	import tarfile
+	import shutil
+
+	s = d.getVar('S',True)
+	work_dir = d.getVar('WORKDIR', True)
+	os.chdir(work_dir)
+	patch_dir = os.path.basename(patchdir)
+	tarname = patch_dir + ".tar.gz"
+	if series  == 'all' and os.path.exists(s + '/patches/series'):
+			shutil.copy(s + '/patches/series',patch_dir)
+	tar = tarfile.open(tarname, "w:gz")
+	tar.add(patch_dir)
+	tar.close()
+	shutil.rmtree(patch_dir, ignore_errors=True)
+	return tarname
+
+def select_archive_patches(d,option):
+	'''select to archive all patches including non-applying and series or applying patches '''
+	if option == "all":
+		patchdir = get_all_patches(d)
+	elif option == "applying":
+		patchdir = get_applying_patches(d)
+	try:
+		os.rmdir(patchdir)
+	except OSError:
+			tarpatch = archive_patches(d,patchdir,option)
+			return tarpatch
+	return
+
+def archive_logs(d,logdir,bbinc=False):
+	'''archive logs in temp to tarball and .bb and .inc files if bbinc is True '''
+	import tarfile
+	import shutil
+
+	log_dir = os.path.basename(logdir)
+	pf = d.getVar('PF',True)
+	tarname = pf + '-' + log_dir + ".tar.gz"
+	tar = tarfile.open(tarname, "w:gz")
+	tar.add(log_dir)
+	tar.close()
+	if bbinc:
+		shutil.rmtree(log_dir, ignore_errors=True)
+	return tarname 
+
+def get_licenses(d):
+	'''get licenses for running .bb file'''
+	licenses = d.getVar('LICENSE', 1).replace('&', '|')
+	licenses = licenses.replace('(', '').replace(')', '') 
+	clean_licenses = ""
+	for x in licenses.split():
+		if x.strip() == '' or x == 'CLOSED':
+			continue
+		if x != "|":
+			clean_licenses += x
+	if '|' in clean_licenses:
+		clean_licenses = clean_licenses.replace('|','')
+	return clean_licenses
+	
+
+def move_tarball_deploy(d,tarball_list):
+	'''move tarball in location to ${DEPLOY_DIR}/sources'''
+	import shutil
+	
+	if tarball_list is []:
+		return
+	target_sys = d.getVar('TARGET_SYS', True)
+	pf = d.getVar('PF', True)
+	licenses = get_licenses(d)
+	tar_sources = d.getVar('DEPLOY_DIR', True) + '/sources/' + target_sys + '/' + licenses + '/' + pf
+	if not os.path.exists(tar_sources):
+		bb.mkdirhier(tar_sources)
+	for source in tarball_list:
+		if source:
+			if os.path.exists(tar_sources + '/' + source):
+				os.remove(tar_sources + '/' + source)
+			shutil.move(source,tar_sources)
+
+def verify_var(d):
+	'''check the type for archiving package('tar' or 'srpm')'''
+	try:
+		if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() not in d.getVar('ARCHIVE_TYPE', True).split():
+			raise AttributeError	
+	except AttributeError:
+			bb.fatal("\"SOURCE_ARCHIVE_PACKAGE_TYPE\" is \'tar\' or \'srpm\', no other types") 
+
+def archive_copyleft(d):
+	'''check if a package to archive is copy-left'''
+	if bb.data.inherits_class('copyleft_compliance', d):
+		included, reason = copyleft_should_include(d)
+		if not included:
+			return False
+		return True
+def archive_sources_patches(d,middle_name):
+	'''archive sources and patches to tarball. middle_name will append strings ${middle_name} to ${PR} as middle name. for example, zlib-1.4.6-prepatch(middle_name).tar.gz '''
+	verify_var(d)	
+	if not_tarball(d):
+		return
+	
+	source_tar_name = archive_sources(d,middle_name)
+	if middle_name == "prepatch":
+		if d.getVar('PATCHES_ARCHIVE_WITH_SERIES',True).upper() == 'TRUE':
+			patch_tar_name = select_archive_patches(d,"all")
+		elif d.getVar('PATCHES_ARCHIVE_WITH_SERIES',True).upper() == 'FALSE':
+			patch_tar_name = select_archive_patches(d,"applying")
+		else:
+			bb.fatal("Please define 'PATCHES_ARCHIVE_WITH_SERIES' is strings 'True' or 'False' ")
+	else:
+		patch_tar_name = ''
+
+	if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() not in 'SRPM':
+		move_tarball_deploy(d,[source_tar_name,patch_tar_name])
+
+def archive_sources_patches_logs_copyleft(d,middle_name):
+	'''archive source, patches and according to the variable "COPYLEFT_COMPLIANCE", If this variable is 'True', then archive the packages for copy-left, or else archive all packages'''
+	verify_var(d)	
+	if not_tarball(d):
+		return
+	copyleft_compliance = d.getVar('COPYLEFT_COMPLIANCE', True)
+	if copyleft_compliance is None:
+		archive_sources_patches(d,middle_name)
+	elif copyleft_compliance.upper() == 'TRUE' and archive_copyleft(d):
+		archive_sources_patches(d,middle_name)
+	
+	
+def archive_scripts_logs(d):
+	'''archive scripts and logs. scripts include .bb and .inc files and logs include stuff in "temp".'''
+
+	s = d.getVar('WORKDIR', True)
+	os.chdir(s)
+	source_archive_log_with_scripts = d.getVar('SOURCE_ARCHIVE_LOG_WITH_SCRIPTS', True)
+	if source_archive_log_with_scripts == 'logs_with_scripts':
+		logdir = get_bb_inc(d)
+		tarlog = archive_logs(d,logdir,True)
+	elif source_archive_log_with_scripts == 'logs':
+		if os.path.exists('temp'):
+			archive_logs(d,'temp',False)
+	else:
+		return
+		
+	if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() not in 'SRPM':
+		move_tarball_deploy(d,[tarlog])
+
+def archive_scripts_logs_copyleft(d):
+	'''archive logs according to the variable "COPYLEFT_COMPLIANCE", If this variable is 'True', then archive the packages for copy-left, or else archive all packages'''
+	verify_var(d)
+	if not d.getVar('SOURCE_ARCHIVE_LOG_WITH_SCRIPTS', True) or not_tarball(d):
+		return
+	copyleft_compliance = d.getVar('COPYLEFT_COMPLIANCE', True)
+	if copyleft_compliance is None:
+			archive_scripts_logs(d)
+	elif copyleft_compliance.upper() == 'TRUE' and archive_copyleft(d):
+			archive_scripts_logs(d)
+		
+
+def dumpdata(d):
+	'''dump environment to "${P}-${PR}.showdata.dump" including all kinds of variables and functions when running a task'''
+	workdir = bb.data.getVar('WORKDIR', d, 1)
+	distro = bb.data.getVar('DISTRO', d, 1)
+	s = d.getVar('S', True)
+	pf = d.getVar('PF', True)
+	target_sys = d.getVar('TARGET_SYS', True)
+	licenses = get_licenses(d)
+	dumpdir = d.getVar('DEPLOY_DIR', True) + '/sources/' + target_sys + '/' + licenses + '/' + pf
+	if not os.path.exists(dumpdir):
+		bb.mkdirhier(dumpdir)
+
+	dumpfile = os.path.join(dumpdir, bb.data.expand("${P}-${PR}.showdata.dump",d))
+
+	bb.note("Dumping metadata into '%s'" % dumpfile)
+	f = open(dumpfile, "w")
+	# emit variables and shell functions
+        bb.data.emit_env(f, d, True)
+	# emit the metadata which isnt valid shell
+	for e in d.keys():
+		if bb.data.getVarFlag(e, 'python', d):
+			f.write("\npython %s () {\n%s}\n" % (e, bb.data.getVar(e, d, 1)))
+	f.close()
+
+
+
+def create_diff_gz(d):
+	'''creating .diff.gz in ${DEPLOY_DIR_SRC}/${P}-${PR}.diff.g gz for mapping all content in 's' including patches to  xxx.diff.gz'''
+	import shutil
+
+	work_dir = d.getVar('WORKDIR', True)
+	exclude_from = d.getVar('EXCLUDE_FROM', True).split()
+	pf = d.getVar('PF', True)
+	licenses = get_licenses(d)
+	target_sys = d.getVar('TARGET_SYS', True)
+	diff_dir = d.getVar('DEPLOY_DIR', True) + '/sources/' + target_sys + '/' + licenses + '/' + pf
+	diff_file = os.path.join(diff_dir, bb.data.expand("${P}-${PR}.diff.gz",d))
+	os.chdir(work_dir)
+	f = open('temp/exclude-from-file', 'a')
+	for i in exclude_from:
+		f.write(i)
+		f.write("\n")
+	f.close()
+
+
+	s=d.getVar('S', True)
+	distro = d.getVar('DISTRO',True)
+	dest = s + '/' + distro + '/files'
+	if not os.path.exists(dest):
+		bb.mkdirhier(dest)
+	for i in os.listdir(os.getcwd()):
+		if os.path.isfile(i):
+			shutil.copy(i, dest)
+	
+	bb.note("Creating .diff.gz in ${DEPLOY_DIR_SRC}/${P}-${PR}.diff.gz")
+	cmd = "LC_ALL=C TZ=UTC0 diff --exclude-from=" + work_dir + "/temp/exclude-from-file -Naur " + s + '.org' + ' ' +  s + " | gzip -c > " + diff_file
+	d.setVar('DIFF', cmd + "\n")
+	d.setVarFlag('DIFF', 'func', '1') 
+	bb.build.exec_func('DIFF', d)
+	shutil.rmtree(s + '.org', ignore_errors=True)
+
+# This function will run when user want to get tarball for sources and patches after do_unpack
+python do_archive_original_sources_patches(){
+	archive_sources_patches_logs_copyleft(d,'prepatch')
+}
+
+# This function will run when user want to get tarball for patched sources after do_patch
+python do_archive_patched_sources(){
+	archive_sources_patches_logs_copyleft(d,'patched')
+}
+
+# This function will run when user want to get tarball for configured sources after do_configure
+python do_archive_configured_sources(){
+	archive_sources_patches_logs_copyleft(d,'configured')
+}
+
+# This function will run when user want to get tarball for logs or both logs and scripts(.bb and .inc files)
+python do_archive_scripts_logs_copyleft(){
+	archive_scripts_logs_copyleft(d)
+}
+
+# This function will run when user want to know what variable and functions in a running task are and also can get a diff file including 
+# all content a package should include.
+python do_dumpdata_create_diff_gz(){
+	dumpdata(d)
+	create_diff_gz(d)
+}
+
+# This functions prepare for archiving "linux-yocto" because this package create directory 's' before do_patch instead of after do_unpack.
+# This is special control for archiving linux-yocto only.
+python do_archive_linux_yocto(){
+	s = d.getVar('S', True)
+	if 'linux-yocto' in s:
+		source_tar_name = archive_sources(d,'')
+	if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() not in 'SRPM':
+		move_tarball_deploy(d,[source_tar_name,''])
+}
+do_kernel_checkout[postfuncs] += "do_archive_linux_yocto "
+
+# remove tarball for sources, patches and logs after creating srpm.
+python do_remove_tarball(){
+	if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() == 'SRPM':
+		work_dir = d.getVar('WORKDIR', True)
+		os.chdir(work_dir)
+		for file in os.listdir(os.getcwd()):
+			if '.tar.gz' in file:
+				os.remove(file)
+}
+do_remove_taball[deptask] = "do_archive_scripts_logs"
+do_package_write_rpm[postfuncs] += "do_remove_tarball "
-- 
1.7.0.4

[PATCH 3/7] archive-patched-source.bbclass: Archive patched source

[Xiaofeng Yan]

From: Xiaofeng Yan <xiaofeng.yan@windriver.com>

This bbclass inherits archiver.bbclass to archive patched source

[#YOCTO 1977]

Signed-off-by: Xiaofeng Yan <xiaofeng.yan@windriver.com>
---
 meta/classes/archive-patched-source.bbclass |   14 ++++++++++++++
 1 files changed, 14 insertions(+), 0 deletions(-)
 create mode 100644 meta/classes/archive-patched-source.bbclass

diff --git a/meta/classes/archive-patched-source.bbclass b/meta/classes/archive-patched-source.bbclass
new file mode 100644
index 0000000..b3184b5
--- /dev/null
+++ b/meta/classes/archive-patched-source.bbclass
@@ -0,0 +1,14 @@
+# This file is for getting archiving packages with patched sources(archive 's' before do_patch stage),logs(archive 'temp' after package_write_rpm),dump data and 
+# creating diff file(get all environment variables and functions in building and mapping all content in 's' including patches to  xxx.diff.gz.
+# All archived packages will be deployed in ${DEPLOY_DIR}/sources
+
+inherit archiver
+
+# Get archiving package with patched sources including patches
+do_patch[postfuncs] += "do_archive_patched_sources "
+
+# Get archiving package with logs(temp) and scripts(.bb and .inc files)
+do_package_write_rpm[prefuncs] += "do_archive_scripts_logs_copyleft "
+
+# Get dump date and create diff file 
+do_package_write_rpm[postfuncs] += "do_dumpdata_create_diff_gz "
-- 
1.7.0.4

[PATCH 4/7] archive-configured-source.bbclass: Archive configured source

[Xiaofeng Yan]

From: Xiaofeng Yan <xiaofeng.yan@windriver.com>

This bbclass prepares for archiving configured source.

[#YOCTO 1977]

Signed-off-by: Xiaofeng Yan <xiaofeng.yan@windriver.com>
---
 meta/classes/archive-configured-source.bbclass |   14 ++++++++++++++
 1 files changed, 14 insertions(+), 0 deletions(-)
 create mode 100644 meta/classes/archive-configured-source.bbclass

diff --git a/meta/classes/archive-configured-source.bbclass b/meta/classes/archive-configured-source.bbclass
new file mode 100644
index 0000000..5f9d444
--- /dev/null
+++ b/meta/classes/archive-configured-source.bbclass
@@ -0,0 +1,14 @@
+# This file is for getting archiving packages with configured sources(archive 's' after configure stage),logs(archive 'temp' after package_write_rpm),dump data 
+# and creating diff file(get all environment variables and functions in building and mapping all content in 's' including patches to  xxx.diff.gz.
+# All archived packages will be deployed in ${DEPLOY_DIR}/sources
+
+inherit archiver
+
+# Get archiving package with configured sources including patches
+do_configure[postfuncs] += "do_archive_configured_sources "
+
+# Get archiving package with temp(logs) and scripts(.bb and inc files)
+do_package_write_rpm[prefuncs] += "do_archive_scripts_logs_copyleft "
+
+# Get dump date and create diff file 
+do_package_write_rpm[postfuncs] += "do_dumpdata_create_diff_gz "
-- 
1.7.0.4

[PATCH 5/7] archive-original-source.bbclass: Archive original source

[Xiaofeng Yan]

From: Xiaofeng Yan <xiaofeng.yan@windriver.com>

This bbclass prepares for archiving original source.

[#YOCTO 1977]

Signed-off-by: Xiaofeng Yan <xiaofeng.yan@windriver.com>
---
 meta/classes/archive-original-source.bbclass |   14 ++++++++++++++
 1 files changed, 14 insertions(+), 0 deletions(-)
 create mode 100644 meta/classes/archive-original-source.bbclass

diff --git a/meta/classes/archive-original-source.bbclass b/meta/classes/archive-original-source.bbclass
new file mode 100644
index 0000000..d58c15f
--- /dev/null
+++ b/meta/classes/archive-original-source.bbclass
@@ -0,0 +1,14 @@
+# This file is for getting archiving packages with original sources(archive 's' after unpack stage),patches,logs(archive 'temp' after package_write_rpm),dump data and 
+# creating diff file(get all environment variables and functions in building and mapping all content in 's' including patches to  xxx.diff.gz.
+# All archived packages will be deployed in ${DEPLOY_DIR}/sources
+
+inherit archiver
+
+# Get original sources archiving package with patches
+do_unpack[postfuncs] += "do_archive_original_sources_patches "
+
+# Get archiving package with temp(logs) and scripts(.bb and inc files)
+do_package_write_rpm[prefuncs] += "do_archive_scripts_logs_copyleft "
+
+# Get dump date and create diff file 
+do_package_write_rpm[postfuncs] += "do_dumpdata_create_diff_gz "
-- 
1.7.0.4

[PATCH 6/7] local.conf.sample: Add set for archiving packages

[Xiaofeng Yan]

From: Xiaofeng Yan <xiaofeng.yan@windriver.com>

User can use these variables to get atchiving packages they want.

[YOCTO #1977]

Signed-off-by: Xiaofeng Yan <xiaofeng.yan@windriver.com>
---
 meta-yocto/conf/local.conf.sample |   25 +++++++++++++++++++++++++
 1 files changed, 25 insertions(+), 0 deletions(-)

diff --git a/meta-yocto/conf/local.conf.sample b/meta-yocto/conf/local.conf.sample
index 38507e3..6f16c9a 100644
--- a/meta-yocto/conf/local.conf.sample
+++ b/meta-yocto/conf/local.conf.sample
@@ -121,6 +121,31 @@ DISTRO ?= "poky"
 PACKAGE_CLASSES ?= "package_rpm"
 
 #
+# Archiving packages configuration
+#
+# The following variables lists which files to archive and the package type to archive.
+# One bbclass of them (archive-original-source.bbclass,archive-patched-source.bbclass 
+# and archive-configured-source.bbclass) should be inherited in a globale bbclass, for example,
+# intherit archive-original-source in packages_rpm.bbclass
+#
+# SOURCE_ARCHIVE_PACKAGE_TYPE = {'tar','srpm'}
+#SOURCE_ARCHIVE_PACKAGE_TYPE ?= 'tar'
+#
+# SOURCE_ARCHIVE_LOG_WITH_SCRIPTS = {'logs_with_scripts', 'logs'}
+# String 'logs_with_scripts' include temp directory and .bb and .inc files
+# String 'logs' only include temp 
+#SOURCE_ARCHIVE_LOG_WITH_SCRIPTS ?= 'logs_with_scripts'
+#
+# PATCHES_ARCHIVE_WITH_SERIES = {'true', 'false'}
+# Strings 'true' means that patches including series files(series + non-applying)
+# String 'false' means that no series and only archive applying patches
+#PATCHES_ARCHIVE_WITH_SERIES ?= 'true'
+#
+# Archive packages for copyleft(GPL*,LGPL*)
+#COPYLEFT_COMPLIANCE ?= 'true'
+#
+
+#
 # SDK/ADT target architecture
 #
 # This variable specified the architecture to build SDK/ADT items for and means
-- 
1.7.0.4

[PATCH 7/7] sourcepkg, src_distribute, src_distribute_local: remove three bbclass

[Xiaofeng Yan]

From: Xiaofeng Yan <xiaofeng.yan@windriver.com>

These bbclasses have been merged into archiver.bbclass.

[YOCTO #1977]

Signed-off-by: Xiaofeng Yan <xiaofeng.yan@windriver.com>
---
 meta/classes/sourcepkg.bbclass            |  107 -----------------------------
 meta/classes/src_distribute.bbclass       |   49 -------------
 meta/classes/src_distribute_local.bbclass |   33 ---------
 3 files changed, 0 insertions(+), 189 deletions(-)
 delete mode 100644 meta/classes/sourcepkg.bbclass
 delete mode 100644 meta/classes/src_distribute.bbclass
 delete mode 100644 meta/classes/src_distribute_local.bbclass

diff --git a/meta/classes/sourcepkg.bbclass b/meta/classes/sourcepkg.bbclass
deleted file mode 100644
index 102c109..0000000
--- a/meta/classes/sourcepkg.bbclass
+++ /dev/null
@@ -1,107 +0,0 @@
-DEPLOY_DIR_SRC ?= "${DEPLOY_DIR}/source"
-EXCLUDE_FROM ?= ".pc autom4te.cache"
-
-# used as part of a path. make sure it's set
-DISTRO ?= "openembedded"
-
-def get_src_tree(d):
-
-	workdir = d.getVar('WORKDIR', True)
-	if not workdir:
-		bb.error("WORKDIR not defined, unable to find source tree.")
-		return
-
-	s = d.getVar('S', 0)
-	if not s:
-		bb.error("S not defined, unable to find source tree.")
-		return
-
-	s_tree_raw = s.split('/')[1]
-	s_tree = d.expand(s_tree_raw)
-
-	src_tree_path = os.path.join(workdir, s_tree)
-	try:
-		os.listdir(src_tree_path)
-	except OSError:
-		bb.fatal("Expected to find source tree in '%s' which doesn't exist." % src_tree_path)
-	bb.debug("Assuming source tree is '%s'" % src_tree_path)
-
-	return s_tree
-
-sourcepkg_do_create_orig_tgz(){
-
-	mkdir -p ${DEPLOY_DIR_SRC}
-	cd ${WORKDIR}
-	for i in ${EXCLUDE_FROM}; do
-		echo $i >> temp/exclude-from-file
-	done
-
-	src_tree=${@get_src_tree(d)}
-	
-	echo $src_tree
-	bbnote "Creating .orig.tar.gz in ${DEPLOY_DIR_SRC}/${P}.orig.tar.gz"
-	tar cvzf ${DEPLOY_DIR_SRC}/${P}.orig.tar.gz --exclude-from temp/exclude-from-file $src_tree
-	tar -cf - -C $src_tree -ps . | tar -xf - -C $src_tree.orig
-}
-
-sourcepkg_do_archive_bb() {
-
-	src_tree=${@get_src_tree(d)}
-	dest=${WORKDIR}/$src_tree/${DISTRO}
-	mkdir -p $dest
-
-	cp ${FILE} $dest
-}
-
-python sourcepkg_do_dumpdata() {
-
-	workdir = d.getVar('WORKDIR', True)
-	distro = d.getVar('DISTRO', True)
-	s_tree = get_src_tree(d)
-	openembeddeddir = os.path.join(workdir, s_tree, distro)
-	dumpfile = os.path.join(openembeddeddir, d.expand("${P}-${PR}.showdata.dump"))
-	
-	try:
-		os.mkdir(openembeddeddir)
-	except OSError:
-		# dir exists
-		pass
-
-	bb.note("Dumping metadata into '%s'" % dumpfile)
-	f = open(dumpfile, "w")
-	# emit variables and shell functions
-        bb.data.emit_env(f, d, True)
-	# emit the metadata which isnt valid shell
-	for e in d.keys():
-		if d.getVarFlag(e, 'python'):
-			f.write("\npython %s () {\n%s}\n" % (e, d.getVar(e, True)))
-	f.close()
-}
-
-sourcepkg_do_create_diff_gz(){
-
-	cd ${WORKDIR}
-	for i in ${EXCLUDE_FROM}; do
-		echo $i >> temp/exclude-from-file
-	done
-
-
-	src_tree=${@get_src_tree(d)}
-
-	for i in `find . -maxdepth 1 -type f`; do
-		mkdir -p $src_tree/${DISTRO}/files
-		cp $i $src_tree/${DISTRO}/files
-	done
-	
-	bbnote "Creating .diff.gz in ${DEPLOY_DIR_SRC}/${P}-${PR}.diff.gz"
-	LC_ALL=C TZ=UTC0 diff --exclude-from=temp/exclude-from-file -Naur $src_tree.orig $src_tree | gzip -c > ${DEPLOY_DIR_SRC}/${P}-${PR}.diff.gz
-	rm -rf $src_tree.orig
-}
-
-EXPORT_FUNCTIONS do_create_orig_tgz do_archive_bb do_dumpdata do_create_diff_gz
-
-addtask create_orig_tgz after do_unpack before do_patch
-addtask archive_bb after do_patch before do_dumpdata
-addtask dumpdata after archive_bb before do_create_diff_gz
-addtask create_diff_gz after do_dump_data before do_configure
-
diff --git a/meta/classes/src_distribute.bbclass b/meta/classes/src_distribute.bbclass
deleted file mode 100644
index efa2720..0000000
--- a/meta/classes/src_distribute.bbclass
+++ /dev/null
@@ -1,49 +0,0 @@
-SRC_DISTRIBUTECOMMAND[func] = "1"
-python do_distribute_sources () {
-	l = bb.data.createCopy(d)
-	bb.data.update_data(l)
-
-	sources_dir = d.getVar('SRC_DISTRIBUTEDIR', True)
-	src_uri = d.getVar('SRC_URI', True).split()
-	fetcher = bb.fetch2.Fetch(src_uri, d)
-	ud = fetcher.ud
-
-	licenses = d.getVar('LICENSE', True).replace('&', '|')
-	licenses = licenses.replace('(', '').replace(')', '')
-	clean_licenses = ""
-	for x in licenses.split():
-		if x.strip() == '' or x == 'CLOSED':
-			continue
-
-		if x != "|":
-			clean_licenses += x
-
-	for license in clean_licenses.split('|'):
-		for url in ud.values():
-			cmd = d.getVar('SRC_DISTRIBUTECOMMAND', True)
-			if not cmd:
-				raise bb.build.FuncFailed("Unable to distribute sources, SRC_DISTRIBUTECOMMAND not defined")
-			url.setup_localpath(d)
-			d.setVar('SRC', url.localpath)
-			if url.type == 'file':
-				if url.basename == '*':
-					import os.path
-					dest_dir = os.path.basename(os.path.dirname(os.path.abspath(url.localpath)))
-					d.setVar('DEST', "%s_%s/" % (d.getVar('PF', True), dest_dir))
-				else:
-					d.setVar('DEST', "%s_%s" % (d.getVar('PF', True), url.basename))
-			else:
-				d.setVar('DEST', '')
-
-			d.setVar('SRC_DISTRIBUTEDIR', "%s/%s" % (sources_dir, license))
-			bb.build.exec_func('SRC_DISTRIBUTECOMMAND', d)
-}
-
-addtask distribute_sources before do_build after do_fetch
-
-addtask distribute_sources_all after do_distribute_sources
-do_distribute_sources_all[recrdeptask] = "do_distribute_sources"
-do_distribute_sources_all[nostamp] = "1"
-do_distribute_sources_all () {
-	:
-}
diff --git a/meta/classes/src_distribute_local.bbclass b/meta/classes/src_distribute_local.bbclass
deleted file mode 100644
index 17b67e3..0000000
--- a/meta/classes/src_distribute_local.bbclass
+++ /dev/null
@@ -1,33 +0,0 @@
-inherit src_distribute
-
-# SRC_DIST_LOCAL possible values:
-# copy		copies the files to the distributedir
-# symlink	symlinks the files to the distributedir
-# move+symlink	moves the files into distributedir, and symlinks them back
-SRC_DIST_LOCAL ?= "move+symlink"
-SRC_DISTRIBUTEDIR ?= "${DEPLOY_DIR}/sources"
-SRC_DISTRIBUTECOMMAND () {
-	s="${SRC}"
-	d="${DEST}"
-
-	mkdir -p ${SRC_DISTRIBUTEDIR}
-
-	if echo $d | grep -q '/$'; then
-		mkdir -p ${SRC_DISTRIBUTEDIR}/$d
-	fi
-
-	case "${SRC_DIST_LOCAL}" in
-		copy)
-			test -e $s.md5 && cp -f $s.md5 ${SRC_DISTRIBUTEDIR}/$d.md5
-			cp -f $s ${SRC_DISTRIBUTEDIR}/$d
-			;;
-		symlink)
-			test -e $s.md5 && ln -sf $s.md5 ${SRC_DISTRIBUTEDIR}/$d.md5
-			ln -sf $s ${SRC_DISTRIBUTEDIR}/$d
-			;;
-		move+symlink)
-			mv $s ${SRC_DISTRIBUTEDIR}/$d
-			ln -sf ${SRC_DISTRIBUTEDIR}/$d $s
-			;;
-	esac
-}
-- 
1.7.0.4

Re: [PATCH 1/7] package_rpm: Add srpm function to this bbclass

[Saul Wold]

On 03/13/2012 05:52 AM, Xiaofeng Yan wrote:
> From: Xiaofeng Yan<xiaofeng.yan@windriver.com>
>
> Add a new function to archive source, patches and logs to a source rpm
> package. Every source rpm package will be deployed to
> ${DEPLOY_DIR}/sources/deploy-srpm.
>
> [YOCTO #1977]
>
> Signed-off-by: Xiaofeng Yan<xiaofeng.yan@windriver.com>
> ---
>   meta/classes/package_rpm.bbclass |   63 ++++++++++++++++++++++++++++++++++++++
>   1 files changed, 63 insertions(+), 0 deletions(-)
>
> diff --git a/meta/classes/package_rpm.bbclass b/meta/classes/package_rpm.bbclass
> index 68313ec..b261a0e 100644
> --- a/meta/classes/package_rpm.bbclass
> +++ b/meta/classes/package_rpm.bbclass
> @@ -6,6 +6,7 @@ RPM="rpm"
>   RPMBUILD="rpmbuild"
>
>   PKGWRITEDIRRPM = "${WORKDIR}/deploy-rpms"
> +PKGWRITEDIRSRPM = "${DEPLOY_DIR}/sources/deploy-srpm"
>
>   python package_rpm_fn () {
>   	d.setVar('PKGFN', d.getVar('PKG'))
> @@ -475,6 +476,37 @@ python write_specfile () {
>   	import textwrap
>   	import oe.packagedata
>
> +	# append information for logs and patches to %prep
> +	def add_prep(d,spec_files_bottom):
> +		if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True) and d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() == 'SRPM':
> +			spec_files_bottom.append('%%prep -n %s' % d.getVar('PN', True) )
> +			spec_files_bottom.append('%s' % "echo \"include logs and patches, Please check them in SOURCES\"")
> +			spec_files_bottom.append('')
> +
> +	# get the name of tarball for sources, patches and logs
> +	def get_tarballs(d):
> +		if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True) and d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() == 'SRPM':
> +			sourcelist=[]
> +			workdir = d.getVar('WORKDIR',True)
> +			print os.listdir(workdir)
> +			for source in os.listdir(workdir):
> +				if 'tar.gz' in source:
> +					sourcelist.append(source)
> +			return sourcelist

What happens here is there are some other tar.gz file in the WORKDIR? 
Is there a way to ensure you just get the tarballs you are looking for?
> +
> +	# append the name of tarball to key word 'SOURCE' in xxx.spec.
> +	def tail_source(d,source_list=[],patch_list=None):
> +		if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True) and d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() == 'SRPM':
> +			source_number = 0
> +			patch_number = 0
> +			for source in source_list:
> +				spec_preamble_top.append('Source' + str(source_number) + ': %s' % source)
> +				source_number += 1
> +			if patch_list:
> +				for patch in patch_list:
> +					print_deps(patch, "Patch" + str(patch_number), spec_preamble_top, d)
> +					patch_number += 1
> +
>   	# We need a simple way to remove the MLPREFIX from the package name,
>   	# and dependency information...
>   	def strip_multilib(name, d):
> @@ -707,6 +739,8 @@ python write_specfile () {
>   			spec_preamble_bottom.append('License: %s' % splitlicense)
>   		spec_preamble_bottom.append('Group: %s' % splitsection)
>
> +		source_list = get_tarballs(d)
> +		tail_source(d,source_list,None)
>   		# Replaces == Obsoletes&&  Provides
>   		if splitrreplaces and splitrreplaces.strip() != "":
>   			for dep in splitrreplaces.split(','):
> @@ -786,6 +820,7 @@ python write_specfile () {
>   		del localdata
>   		bb.utils.unlockfile(lf)
>
> +	add_prep(d,spec_files_bottom)
>   	spec_preamble_top.append('Summary: %s' % srcsummary)
>   	spec_preamble_top.append('Name: %s' % srcname)
>   	spec_preamble_top.append('Version: %s' % srcversion)
> @@ -900,6 +935,25 @@ python write_specfile () {
>   python do_package_rpm () {
>   	import os
>
> +	def creat_srpm_dir(d):
> +		if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True) and d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() == 'SRPM':
> +			licenses = d.getVar('LICENSE', 1).replace('&', '|')
> +			licenses = licenses.replace('(', '').replace(')', '')
> +			clean_licenses = ""
> +			for x in licenses.split():
> +				if x.strip() == '' or x == 'CLOSED':
What about "Proprietary"
> +					continue
> +				if x != "|":
> +					clean_licenses += x
> +				if '|' in clean_licenses:
> +					clean_licenses = clean_licenses.replace('|','')
> +
Maybe this spinet should be a function in license.bbclass? Maybe it's 
there already, or is this too RPM specific?

> +			pkgwritesrpmdir = bb.data.expand('${PKGWRITEDIRSRPM}/${PACKAGE_ARCH_EXTEND}', d)
> +			pkgwritesrpmdir = pkgwritesrpmdir + '/' + clean_licenses
> +			bb.mkdirhier(pkgwritesrpmdir)
> +			os.chmod(pkgwritesrpmdir, 0755)
> +			return pkgwritesrpmdir
> +
>   	# We need a simple way to remove the MLPREFIX from the package name,
>   	# and dependency information...
>   	def strip_multilib(name, d):
> @@ -1015,8 +1069,17 @@ python do_package_rpm () {
>   	cmd = cmd + " --define 'debug_package %{nil}'"
>   	cmd = cmd + " --define '_rpmfc_magic_path " + magicfile + "'"
>   	cmd = cmd + " --define '_tmppath " + workdir + "'"
> +	if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True) and d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() == 'SRPM':
> +		cmdsrpm = cmd + " --define '_sourcedir " + workdir + "' --define '_srcrpmdir " + creat_srpm_dir(d) + "'"
> +		cmdsrpm = 'fakeroot ' + cmdsrpm + " -bs " + outspecfile
>   	cmd = cmd + " -bb " + outspecfile
>
> +	# Build the source rpm package !
> +	if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True) and d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() == 'SRPM':
> +		d.setVar('SBUILDSPEC', cmdsrpm + "\n")
> +		d.setVarFlag('SBUILDSPEC', 'func', '1')
> +		bb.build.exec_func('SBUILDSPEC', d)
> +
>   	# Build the rpm package!
>   	d.setVar('BUILDSPEC', cmd + "\n")
>   	d.setVarFlag('BUILDSPEC', 'func', '1')

Re: [PATCH 2/7] archiver.bbclass: New bbclass for archiving sources, patches, logs and scripts

[Saul Wold]

On 03/13/2012 05:52 AM, Xiaofeng Yan wrote:
> From: Xiaofeng Yan<xiaofeng.yan@windriver.com>
>
> 1 Archive sources in ${S} in the different stage (do_unpack,do_patch,do_configure).
> 2 Archive patches including series
> 3 Archive logs including scripts (.bb and .inc files)
> 4 dump environment resources which show all variable and
>    functions used to xxx.showdata.dump when running a task
> 5 dump all content in 's' including patches to file xxx.diff.gz
>
> All archiving packages  will be deployed to ${DEPLOY_DIR}/sources/
>
> [#YOCTO 1977]
>
Xiaofeng,

I am making a first pass on this, there might be more, after I use it to 
create archives.

Sau!

> Signed-off-by: Xiaofeng Yan<xiaofeng.yan@windriver.com>
> ---
>   meta/classes/archiver.bbclass |  421 +++++++++++++++++++++++++++++++++++++++++
>   1 files changed, 421 insertions(+), 0 deletions(-)
>   create mode 100644 meta/classes/archiver.bbclass
>
> diff --git a/meta/classes/archiver.bbclass b/meta/classes/archiver.bbclass
> new file mode 100644
> index 0000000..95e870d
> --- /dev/null
> +++ b/meta/classes/archiver.bbclass
> @@ -0,0 +1,421 @@
> +# This file is used for archiving sources ,patches,and logs to tarball.
> +# It also output building environment to xxx.dump.data and create xxx.diff.gz to record
> +# all content in ${S} to a diff file.
> +
> +EXCLUDE_FROM ?= ".pc autom4te.cache"
> +ARCHIVE_TYPE ?= "TAR SRPM"
> +DISTRO ?= "poky"
> +
> +def parse_var(d,var):
> +	''' parse variable like ${PV}  in "require xxx_${PV}.inc"  to a real value. for example, change "require xxx_${PV}.inc" to "require xxx_1.2.inc" '''
> +	import re
> +	pat = re.compile('.*\$({(.*)}).*')
> +	if '$' not in var and '/' not in var:
> +		return var
> +	else:
> +		if '/' in var:
> +			return [i for i in var.split('/') if i.endswith('.inc')][0]
> +		elif '$' in  var:
> +			m = pat.match(var)
> +			patstr = '\$' + m.group(1)
> +			var_str = m.group(2)
> +			return re.sub(patstr,d.getVar(var_str,True),var)
> +		else:
> +			return var
> +		
> +
> +def get_bb_inc(d):
> +	'''create a directory "script-logs" including .bb and .inc file in ${WORKDIR}'''
> +	import re
> +	import os
> +	import shutil
> +	
> +	bbinc = []
> +	pat=re.compile('require\s*([^\s]*\.*)(.*)')
> +	file_dir = d.getVar('FILE', True)
> +	bbdir = os.path.dirname(file_dir)
> +	work_dir = d.getVar('WORKDIR', True)
> +	os.chdir(work_dir)
> +	bb.mkdirhier("script-logs")
> +	os.chdir(bbdir)
> +	bbfile = os.path.basename(file_dir)
> +	bbinc.append(bbfile)
> +	
> +	def get_inc (file):
> +		f = open(file,'r')
> +		for line in f.readlines():
> +			if 'require' not  in line:
> +				bbinc.append(file)
> +			else:
> +				try:
> +					incfile = pat.match(line).group(1)
> +					incfile = parse_var(d,incfile)
> +					bbinc.append(incfile)
> +					get_inc(incfile)
> +				except AttributeError:
> +					pass
> +	get_inc(bbfile)
> +	os.chdir(work_dir)
> +	for root, dirs, files in os.walk(bbdir):
> +		for file in bbinc:
> +			if file in files:
> +				shutil.copy(root + '/' + file,'script-logs')
> +	oe.path.copytree('temp', 'script-logs')
> +	return work_dir + '/script-logs'
> +
> +def get_all_patches(d):
> +	'''copy patches and series file to a pointed directory which will be archived to tarball in ${WORKDIR}'''
> +	import shutil
> +	
> +	src_patches=[]
> +	pf = d.getVar('PF', True)
> +	work_dir = d.getVar('WORKDIR', True)
> +	dest = os.path.join(work_dir, pf + '-patches')
> +	shutil.rmtree(dest, ignore_errors=True)
> +	bb.mkdirhier(dest)
> +	
> +	src_uri = d.getVar('SRC_URI', 1).split()
> +	fetch = bb.fetch2.Fetch(src_uri, d)
> +	locals = (fetch.localpath(url) for url in fetch.urls)
> +	for local in locals:
> +		src_patches.append(local)
> +	for patch in src_patches[1:]:
> +		 shutil.copy(patch,dest)
> +	return dest
> +
> +def get_applying_patches(d):
> +	"""only copy applying patches to a pointed directory which will be archived to tarball"""
> +	import os
> +	import shutil
> +
> +
> +	pf = d.getVar('PF', True)
> +	work_dir = d.getVar('WORKDIR', True)
> +	dest = os.path.join(work_dir, pf + '-patches')
> +	shutil.rmtree(dest, ignore_errors=True)
> +	bb.mkdirhier(dest)
> +
> +
> +	patches = src_patches(d)
> +	for patch in patches:
> +		_, _, local, _, _, parm = bb.decodeurl(patch)
> +		if local:
> +			 shutil.copy(local,dest)
> +	return dest
> +
> +def not_tarball(d):
> +	'''packages including key words 'work-shared','native', 'task-' will be passed'''
> +	import os
> +
> +	workdir = d.getVar('WORKDIR',True)
> +	s = d.getVar('S',True)
> +	if 'work-shared' in s or 'task-' in workdir or 'native' in workdir:
> +		return True
> +	else:
> +		return False
> +
> +def archive_sources(d,middle_name):
> +	'''archive sources codes tree to tarball'''
> +	import tarfile
> +	import shutil
> +	
> +	s = d.getVar('S',True)
> +	workdir=d.getVar('WORKDIR', True)
> +	PF = d.getVar('PF',True)
> +	tarname = PF + '-' + middle_name + ".tar.gz"
> +
> +	if os.path.exists(s) and s is not workdir:
> +		sourcedir = os.path.basename(s)
> +		tarbase = os.path.dirname(s)
> +		if not sourcedir or os.path.dirname(tarbase) == workdir:
> +			sourcedir = os.path.basename(os.path.dirname(s))
> +			tarbase = os.path.dirname(os.path.dirname(s))
> +		os.chdir(tarbase)
> +	else:
> +		sourcedir = os.path.basename(s)
> +		if not os.path.exists(sourcedir):
> +			os.mkdir(sourcedir)
> +		try:
> +			for file in os.listdir(s):
> +				if file is not 'temp' and file is not sourcedir:
> +					shutil.copy(file,sourcedir)
> +		except (IOError,OSError):
> +			pass
> +
> +
> +	if (len(os.listdir(sourcedir))) != 0:
> +		tar = tarfile.open( tarname, "w:gz")
> +		tar.add(sourcedir)
> +		tar.close()
> +		if cmp(workdir,os.path.dirname(s)) and not os.path.exists(workdir + '/' + tarname):
> +			shutil.move(os.path.dirname(s) + '/' + tarname,workdir)
> +	else:
> +		return
> +	return tarname
> +
> +def archive_patches(d,patchdir,series):
> +	'''archive patches to tarball and also include series files if 'series' is True'''
> +	import tarfile
> +	import shutil
> +
> +	s = d.getVar('S',True)
> +	work_dir = d.getVar('WORKDIR', True)
> +	os.chdir(work_dir)
> +	patch_dir = os.path.basename(patchdir)
> +	tarname = patch_dir + ".tar.gz"
> +	if series  == 'all' and os.path.exists(s + '/patches/series'):
> +			shutil.copy(s + '/patches/series',patch_dir)
> +	tar = tarfile.open(tarname, "w:gz")
> +	tar.add(patch_dir)
> +	tar.close()
> +	shutil.rmtree(patch_dir, ignore_errors=True)
> +	return tarname
> +
> +def select_archive_patches(d,option):
> +	'''select to archive all patches including non-applying and series or applying patches '''
> +	if option == "all":
> +		patchdir = get_all_patches(d)
> +	elif option == "applying":
> +		patchdir = get_applying_patches(d)
> +	try:
> +		os.rmdir(patchdir)
> +	except OSError:
> +			tarpatch = archive_patches(d,patchdir,option)
> +			return tarpatch
> +	return
> +
> +def archive_logs(d,logdir,bbinc=False):
> +	'''archive logs in temp to tarball and .bb and .inc files if bbinc is True '''
> +	import tarfile
> +	import shutil
> +
> +	log_dir = os.path.basename(logdir)
> +	pf = d.getVar('PF',True)
> +	tarname = pf + '-' + log_dir + ".tar.gz"
> +	tar = tarfile.open(tarname, "w:gz")
> +	tar.add(log_dir)
> +	tar.close()
> +	if bbinc:
> +		shutil.rmtree(log_dir, ignore_errors=True)
> +	return tarname
> +
> +def get_licenses(d):
> +	'''get licenses for running .bb file'''
> +	licenses = d.getVar('LICENSE', 1).replace('&', '|')
> +	licenses = licenses.replace('(', '').replace(')', '')
> +	clean_licenses = ""
> +	for x in licenses.split():
> +		if x.strip() == '' or x == 'CLOSED':
> +			continue
> +		if x != "|":
> +			clean_licenses += x
> +	if '|' in clean_licenses:
> +		clean_licenses = clean_licenses.replace('|','')
> +	return clean_licenses
> +	
This seems to be duplicated in your RPM code also?  Move to 
licenses.bbclass maybe?

> +
> +def move_tarball_deploy(d,tarball_list):
> +	'''move tarball in location to ${DEPLOY_DIR}/sources'''
> +	import shutil
> +	
> +	if tarball_list is []:
> +		return
> +	target_sys = d.getVar('TARGET_SYS', True)
> +	pf = d.getVar('PF', True)
> +	licenses = get_licenses(d)
> +	tar_sources = d.getVar('DEPLOY_DIR', True) + '/sources/' + target_sys + '/' + licenses + '/' + pf
> +	if not os.path.exists(tar_sources):
> +		bb.mkdirhier(tar_sources)
> +	for source in tarball_list:
> +		if source:
> +			if os.path.exists(tar_sources + '/' + source):
> +				os.remove(tar_sources + '/' + source)
> +			shutil.move(source,tar_sources)
> +
> +def verify_var(d):
> +	'''check the type for archiving package('tar' or 'srpm')'''
> +	try:
> +		if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() not in d.getVar('ARCHIVE_TYPE', True).split():
> +			raise AttributeError	
> +	except AttributeError:
> +			bb.fatal("\"SOURCE_ARCHIVE_PACKAGE_TYPE\" is \'tar\' or \'srpm\', no other types")
> +
> +def archive_copyleft(d):
> +	'''check if a package to archive is copy-left'''
> +	if bb.data.inherits_class('copyleft_compliance', d):
> +		included, reason = copyleft_should_include(d)
> +		if not included:
> +			return False
> +		return True
> +def archive_sources_patches(d,middle_name):
> +	'''archive sources and patches to tarball. middle_name will append strings ${middle_name} to ${PR} as middle name. for example, zlib-1.4.6-prepatch(middle_name).tar.gz '''
> +	verify_var(d)	
> +	if not_tarball(d):
> +		return
> +	
> +	source_tar_name = archive_sources(d,middle_name)
> +	if middle_name == "prepatch":
> +		if d.getVar('PATCHES_ARCHIVE_WITH_SERIES',True).upper() == 'TRUE':
> +			patch_tar_name = select_archive_patches(d,"all")
> +		elif d.getVar('PATCHES_ARCHIVE_WITH_SERIES',True).upper() == 'FALSE':
> +			patch_tar_name = select_archive_patches(d,"applying")
> +		else:
> +			bb.fatal("Please define 'PATCHES_ARCHIVE_WITH_SERIES' is strings 'True' or 'False' ")
> +	else:
> +		patch_tar_name = ''
> +
> +	if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() not in 'SRPM':
> +		move_tarball_deploy(d,[source_tar_name,patch_tar_name])
> +
> +def archive_sources_patches_logs_copyleft(d,middle_name):
> +	'''archive source, patches and according to the variable "COPYLEFT_COMPLIANCE", If this variable is 'True', then archive the packages for copy-left, or else archive all packages'''
> +	verify_var(d)	
> +	if not_tarball(d):
> +		return
> +	copyleft_compliance = d.getVar('COPYLEFT_COMPLIANCE', True)
> +	if copyleft_compliance is None:
> +		archive_sources_patches(d,middle_name)
> +	elif copyleft_compliance.upper() == 'TRUE' and archive_copyleft(d):
> +		archive_sources_patches(d,middle_name)
> +	
> +	
> +def archive_scripts_logs(d):
> +	'''archive scripts and logs. scripts include .bb and .inc files and logs include stuff in "temp".'''
> +
> +	s = d.getVar('WORKDIR', True)
> +	os.chdir(s)
> +	source_archive_log_with_scripts = d.getVar('SOURCE_ARCHIVE_LOG_WITH_SCRIPTS', True)
> +	if source_archive_log_with_scripts == 'logs_with_scripts':
> +		logdir = get_bb_inc(d)
> +		tarlog = archive_logs(d,logdir,True)
> +	elif source_archive_log_with_scripts == 'logs':
> +		if os.path.exists('temp'):
> +			archive_logs(d,'temp',False)
> +	else:
> +		return
> +		
> +	if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() not in 'SRPM':
> +		move_tarball_deploy(d,[tarlog])
> +
> +def archive_scripts_logs_copyleft(d):
> +	'''archive logs according to the variable "COPYLEFT_COMPLIANCE", If this variable is 'True', then archive the packages for copy-left, or else archive all packages'''
> +	verify_var(d)
> +	if not d.getVar('SOURCE_ARCHIVE_LOG_WITH_SCRIPTS', True) or not_tarball(d):
> +		return
> +	copyleft_compliance = d.getVar('COPYLEFT_COMPLIANCE', True)
> +	if copyleft_compliance is None:
> +			archive_scripts_logs(d)
> +	elif copyleft_compliance.upper() == 'TRUE' and archive_copyleft(d):
> +			archive_scripts_logs(d)
> +		
> +
> +def dumpdata(d):
> +	'''dump environment to "${P}-${PR}.showdata.dump" including all kinds of variables and functions when running a task'''
> +	workdir = bb.data.getVar('WORKDIR', d, 1)
> +	distro = bb.data.getVar('DISTRO', d, 1)
> +	s = d.getVar('S', True)
> +	pf = d.getVar('PF', True)
> +	target_sys = d.getVar('TARGET_SYS', True)
> +	licenses = get_licenses(d)
> +	dumpdir = d.getVar('DEPLOY_DIR', True) + '/sources/' + target_sys + '/' + licenses + '/' + pf
> +	if not os.path.exists(dumpdir):
> +		bb.mkdirhier(dumpdir)
> +
> +	dumpfile = os.path.join(dumpdir, bb.data.expand("${P}-${PR}.showdata.dump",d))
> +
> +	bb.note("Dumping metadata into '%s'" % dumpfile)
> +	f = open(dumpfile, "w")
> +	# emit variables and shell functions
> +        bb.data.emit_env(f, d, True)
> +	# emit the metadata which isnt valid shell
> +	for e in d.keys():
> +		if bb.data.getVarFlag(e, 'python', d):
> +			f.write("\npython %s () {\n%s}\n" % (e, bb.data.getVar(e, d, 1)))
> +	f.close()
> +
> +
> +
> +def create_diff_gz(d):
> +	'''creating .diff.gz in ${DEPLOY_DIR_SRC}/${P}-${PR}.diff.g gz for mapping all content in 's' including patches to  xxx.diff.gz'''
> +	import shutil
> +
> +	work_dir = d.getVar('WORKDIR', True)
> +	exclude_from = d.getVar('EXCLUDE_FROM', True).split()
> +	pf = d.getVar('PF', True)
> +	licenses = get_licenses(d)
> +	target_sys = d.getVar('TARGET_SYS', True)
> +	diff_dir = d.getVar('DEPLOY_DIR', True) + '/sources/' + target_sys + '/' + licenses + '/' + pf
> +	diff_file = os.path.join(diff_dir, bb.data.expand("${P}-${PR}.diff.gz",d))
> +	os.chdir(work_dir)
> +	f = open('temp/exclude-from-file', 'a')
> +	for i in exclude_from:
> +		f.write(i)
> +		f.write("\n")
> +	f.close()
> +
> +
> +	s=d.getVar('S', True)
> +	distro = d.getVar('DISTRO',True)
> +	dest = s + '/' + distro + '/files'
> +	if not os.path.exists(dest):
> +		bb.mkdirhier(dest)
> +	for i in os.listdir(os.getcwd()):
> +		if os.path.isfile(i):
> +			shutil.copy(i, dest)
> +	
> +	bb.note("Creating .diff.gz in ${DEPLOY_DIR_SRC}/${P}-${PR}.diff.gz")
> +	cmd = "LC_ALL=C TZ=UTC0 diff --exclude-from=" + work_dir + "/temp/exclude-from-file -Naur " + s + '.org' + ' ' +  s + " | gzip -c>  " + diff_file
> +	d.setVar('DIFF', cmd + "\n")
> +	d.setVarFlag('DIFF', 'func', '1')
> +	bb.build.exec_func('DIFF', d)
> +	shutil.rmtree(s + '.org', ignore_errors=True)
> +
> +# This function will run when user want to get tarball for sources and patches after do_unpack
> +python do_archive_original_sources_patches(){
> +	archive_sources_patches_logs_copyleft(d,'prepatch')
> +}
> +
> +# This function will run when user want to get tarball for patched sources after do_patch
> +python do_archive_patched_sources(){
> +	archive_sources_patches_logs_copyleft(d,'patched')
> +}
> +
> +# This function will run when user want to get tarball for configured sources after do_configure
> +python do_archive_configured_sources(){
> +	archive_sources_patches_logs_copyleft(d,'configured')
> +}
> +
> +# This function will run when user want to get tarball for logs or both logs and scripts(.bb and .inc files)
> +python do_archive_scripts_logs_copyleft(){
> +	archive_scripts_logs_copyleft(d)
> +}
> +
> +# This function will run when user want to know what variable and functions in a running task are and also can get a diff file including
> +# all content a package should include.
> +python do_dumpdata_create_diff_gz(){
> +	dumpdata(d)
> +	create_diff_gz(d)
> +}
> +
> +# This functions prepare for archiving "linux-yocto" because this package create directory 's' before do_patch instead of after do_unpack.
> +# This is special control for archiving linux-yocto only.
> +python do_archive_linux_yocto(){
> +	s = d.getVar('S', True)
> +	if 'linux-yocto' in s:
> +		source_tar_name = archive_sources(d,'')
> +	if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() not in 'SRPM':
> +		move_tarball_deploy(d,[source_tar_name,''])
> +}
> +do_kernel_checkout[postfuncs] += "do_archive_linux_yocto "
> +
> +# remove tarball for sources, patches and logs after creating srpm.
> +python do_remove_tarball(){
> +	if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() == 'SRPM':
> +		work_dir = d.getVar('WORKDIR', True)
> +		os.chdir(work_dir)
> +		for file in os.listdir(os.getcwd()):
> +			if '.tar.gz' in file:
> +				os.remove(file)
As I mentioned in the RPM code, this might not be the best way to do 
this, and it could break builds if you remove a tarball that's expected 
to be there.

Maybe store the tarballs in a ${WORKDIR}/.. directory (same level as temp).

> +}
> +do_remove_taball[deptask] = "do_archive_scripts_logs"
> +do_package_write_rpm[postfuncs] += "do_remove_tarball "

Re: [PATCH 6/7] local.conf.sample: Add set for archiving packages

[Saul Wold]

On 03/13/2012 05:52 AM, Xiaofeng Yan wrote:
> From: Xiaofeng Yan<xiaofeng.yan@windriver.com>
>
> User can use these variables to get atchiving packages they want.
>
> [YOCTO #1977]
>
> Signed-off-by: Xiaofeng Yan<xiaofeng.yan@windriver.com>
> ---
>   meta-yocto/conf/local.conf.sample |   25 +++++++++++++++++++++++++
>   1 files changed, 25 insertions(+), 0 deletions(-)
>
> diff --git a/meta-yocto/conf/local.conf.sample b/meta-yocto/conf/local.conf.sample
> index 38507e3..6f16c9a 100644
> --- a/meta-yocto/conf/local.conf.sample
> +++ b/meta-yocto/conf/local.conf.sample
> @@ -121,6 +121,31 @@ DISTRO ?= "poky"
>   PACKAGE_CLASSES ?= "package_rpm"
>
>   #
> +# Archiving packages configuration
> +#
> +# The following variables lists which files to archive and the package type to archive.
> +# One bbclass of them (archive-original-source.bbclass,archive-patched-source.bbclass
> +# and archive-configured-source.bbclass) should be inherited in a globale bbclass, for example,
> +# intherit archive-original-source in packages_rpm.bbclass
> +#
> +# SOURCE_ARCHIVE_PACKAGE_TYPE = {'tar','srpm'}
> +#SOURCE_ARCHIVE_PACKAGE_TYPE ?= 'tar'
> +#
> +# SOURCE_ARCHIVE_LOG_WITH_SCRIPTS = {'logs_with_scripts', 'logs'}
> +# String 'logs_with_scripts' include temp directory and .bb and .inc files
> +# String 'logs' only include temp
> +#SOURCE_ARCHIVE_LOG_WITH_SCRIPTS ?= 'logs_with_scripts'
> +#
> +# PATCHES_ARCHIVE_WITH_SERIES = {'true', 'false'}
> +# Strings 'true' means that patches including series files(series + non-applying)
> +# String 'false' means that no series and only archive applying patches
> +#PATCHES_ARCHIVE_WITH_SERIES ?= 'true'
> +#
Why the distinction?  Also, you seem to translate this to all or 
applying, so why not just use those terms here and always include the 
series file?

The PATCHES_ARCHIVE_WITH_SERIES does not really make sense when 
non-applying is added in.

> +# Archive packages for copyleft(GPL*,LGPL*)
> +#COPYLEFT_COMPLIANCE ?= 'true'
> +#
> +
> +#
>   # SDK/ADT target architecture
>   #
>   # This variable specified the architecture to build SDK/ADT items for and means

Re: [PATCH 6/7] local.conf.sample: Add set for archiving packages

[Xiaofeng Yan]

On 2012年03月14日 10:57, Saul Wold wrote:
> On 03/13/2012 05:52 AM, Xiaofeng Yan wrote:
>> From: Xiaofeng Yan<xiaofeng.yan@windriver.com>
>>
>> User can use these variables to get atchiving packages they want.
>>
>> [YOCTO #1977]
>>
>> Signed-off-by: Xiaofeng Yan<xiaofeng.yan@windriver.com>
>> ---
>> meta-yocto/conf/local.conf.sample | 25 +++++++++++++++++++++++++
>> 1 files changed, 25 insertions(+), 0 deletions(-)
>>
>> diff --git a/meta-yocto/conf/local.conf.sample 
>> b/meta-yocto/conf/local.conf.sample
>> index 38507e3..6f16c9a 100644
>> --- a/meta-yocto/conf/local.conf.sample
>> +++ b/meta-yocto/conf/local.conf.sample
>> @@ -121,6 +121,31 @@ DISTRO ?= "poky"
>> PACKAGE_CLASSES ?= "package_rpm"
>>
>> #
>> +# Archiving packages configuration
>> +#
>> +# The following variables lists which files to archive and the 
>> package type to archive.
>> +# One bbclass of them 
>> (archive-original-source.bbclass,archive-patched-source.bbclass
>> +# and archive-configured-source.bbclass) should be inherited in a 
>> globale bbclass, for example,
>> +# intherit archive-original-source in packages_rpm.bbclass
>> +#
>> +# SOURCE_ARCHIVE_PACKAGE_TYPE = {'tar','srpm'}
>> +#SOURCE_ARCHIVE_PACKAGE_TYPE ?= 'tar'
>> +#
>> +# SOURCE_ARCHIVE_LOG_WITH_SCRIPTS = {'logs_with_scripts', 'logs'}
>> +# String 'logs_with_scripts' include temp directory and .bb and .inc 
>> files
>> +# String 'logs' only include temp
>> +#SOURCE_ARCHIVE_LOG_WITH_SCRIPTS ?= 'logs_with_scripts'
>> +#
>> +# PATCHES_ARCHIVE_WITH_SERIES = {'true', 'false'}
>> +# Strings 'true' means that patches including series files(series + 
>> non-applying)
>> +# String 'false' means that no series and only archive applying patches
>> +#PATCHES_ARCHIVE_WITH_SERIES ?= 'true'
>> +#
> Why the distinction? Also, you seem to translate this to all or 
> applying, so why not just use those terms here and always include the 
> series file?
>
I will remove this set and modify archiver.bbclass to including series 
as the default set.
> The PATCHES_ARCHIVE_WITH_SERIES does not really make sense when 
> non-applying is added in.
>> +# Archive packages for copyleft(GPL*,LGPL*)
>> +#COPYLEFT_COMPLIANCE ?= 'true'
>> +#
>> +
>> +#
>> # SDK/ADT target architecture
>> #
>> # This variable specified the architecture to build SDK/ADT items for 
>> and means
>

Re: [PATCH 1/7] package_rpm: Add srpm function to this bbclass

[Xiaofeng Yan]

Hi Saul,

Thanks for your detailed comment
On 2012年03月14日 07:24, Saul Wold wrote:
> On 03/13/2012 05:52 AM, Xiaofeng Yan wrote:
>> From: Xiaofeng Yan<xiaofeng.yan@windriver.com>
>>
>> Add a new function to archive source, patches and logs to a source rpm
>> package. Every source rpm package will be deployed to
>> ${DEPLOY_DIR}/sources/deploy-srpm.
>>
>> [YOCTO #1977]
>>
>> Signed-off-by: Xiaofeng Yan<xiaofeng.yan@windriver.com>
>> ---
>> meta/classes/package_rpm.bbclass | 63 
>> ++++++++++++++++++++++++++++++++++++++
>> 1 files changed, 63 insertions(+), 0 deletions(-)
>>
>> diff --git a/meta/classes/package_rpm.bbclass 
>> b/meta/classes/package_rpm.bbclass
>> index 68313ec..b261a0e 100644
>> --- a/meta/classes/package_rpm.bbclass
>> +++ b/meta/classes/package_rpm.bbclass
>> @@ -6,6 +6,7 @@ RPM="rpm"
>> RPMBUILD="rpmbuild"
>>
>> PKGWRITEDIRRPM = "${WORKDIR}/deploy-rpms"
>> +PKGWRITEDIRSRPM = "${DEPLOY_DIR}/sources/deploy-srpm"
>>
>> python package_rpm_fn () {
>> d.setVar('PKGFN', d.getVar('PKG'))
>> @@ -475,6 +476,37 @@ python write_specfile () {
>> import textwrap
>> import oe.packagedata
>>
>> + # append information for logs and patches to %prep
>> + def add_prep(d,spec_files_bottom):
>> + if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True) and 
>> d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() == 'SRPM':
>> + spec_files_bottom.append('%%prep -n %s' % d.getVar('PN', True) )
>> + spec_files_bottom.append('%s' % "echo \"include logs and patches, 
>> Please check them in SOURCES\"")
>> + spec_files_bottom.append('')
>> +
>> + # get the name of tarball for sources, patches and logs
>> + def get_tarballs(d):
>> + if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True) and 
>> d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() == 'SRPM':
>> + sourcelist=[]
>> + workdir = d.getVar('WORKDIR',True)
>> + print os.listdir(workdir)
>> + for source in os.listdir(workdir):
>> + if 'tar.gz' in source:
>> + sourcelist.append(source)
>> + return sourcelist
>
> What happens here is there are some other tar.gz file in the WORKDIR? 
> Is there a way to ensure you just get the tarballs you are looking for?
It is reasonable for your query. I need add more information to select 
the tarballs srpm wants instead of all.
>> +
>> + # append the name of tarball to key word 'SOURCE' in xxx.spec.
>> + def tail_source(d,source_list=[],patch_list=None):
>> + if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True) and 
>> d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() == 'SRPM':
>> + source_number = 0
>> + patch_number = 0
>> + for source in source_list:
>> + spec_preamble_top.append('Source' + str(source_number) + ': %s' % 
>> source)
>> + source_number += 1
>> + if patch_list:
>> + for patch in patch_list:
>> + print_deps(patch, "Patch" + str(patch_number), spec_preamble_top, d)
>> + patch_number += 1
>> +
>> # We need a simple way to remove the MLPREFIX from the package name,
>> # and dependency information...
>> def strip_multilib(name, d):
>> @@ -707,6 +739,8 @@ python write_specfile () {
>> spec_preamble_bottom.append('License: %s' % splitlicense)
>> spec_preamble_bottom.append('Group: %s' % splitsection)
>>
>> + source_list = get_tarballs(d)
>> + tail_source(d,source_list,None)
>> # Replaces == Obsoletes&& Provides
>> if splitrreplaces and splitrreplaces.strip() != "":
>> for dep in splitrreplaces.split(','):
>> @@ -786,6 +820,7 @@ python write_specfile () {
>> del localdata
>> bb.utils.unlockfile(lf)
>>
>> + add_prep(d,spec_files_bottom)
>> spec_preamble_top.append('Summary: %s' % srcsummary)
>> spec_preamble_top.append('Name: %s' % srcname)
>> spec_preamble_top.append('Version: %s' % srcversion)
>> @@ -900,6 +935,25 @@ python write_specfile () {
>> python do_package_rpm () {
>> import os
>>
>> + def creat_srpm_dir(d):
>> + if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True) and 
>> d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() == 'SRPM':
>> + licenses = d.getVar('LICENSE', 1).replace('&', '|')
>> + licenses = licenses.replace('(', '').replace(')', '')
>> + clean_licenses = ""
>> + for x in licenses.split():
>> + if x.strip() == '' or x == 'CLOSED':
> What about "Proprietary"
the purpose of doing this is for using licenses as a directory. So I 
think x == 'CLOSED' should be removed instead of adding this case. So 
Proprietary does.
>> + continue
>> + if x != "|":
>> + clean_licenses += x
>> + if '|' in clean_licenses:
>> + clean_licenses = clean_licenses.replace('|','')
>> +
> Maybe this spinet should be a function in license.bbclass? Maybe it's 
> there already, or is this too RPM specific?
>
if 'I' is a part of words from licenses, we should replace it. for 
example, this case happen in LICENSE = "Artistic|GPL" from perl_5.14.2.bb
>> + pkgwritesrpmdir = 
>> bb.data.expand('${PKGWRITEDIRSRPM}/${PACKAGE_ARCH_EXTEND}', d)
>> + pkgwritesrpmdir = pkgwritesrpmdir + '/' + clean_licenses
>> + bb.mkdirhier(pkgwritesrpmdir)
>> + os.chmod(pkgwritesrpmdir, 0755)
>> + return pkgwritesrpmdir
>> +
>> # We need a simple way to remove the MLPREFIX from the package name,
>> # and dependency information...
>> def strip_multilib(name, d):
>> @@ -1015,8 +1069,17 @@ python do_package_rpm () {
>> cmd = cmd + " --define 'debug_package %{nil}'"
>> cmd = cmd + " --define '_rpmfc_magic_path " + magicfile + "'"
>> cmd = cmd + " --define '_tmppath " + workdir + "'"
>> + if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True) and 
>> d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() == 'SRPM':
>> + cmdsrpm = cmd + " --define '_sourcedir " + workdir + "' --define 
>> '_srcrpmdir " + creat_srpm_dir(d) + "'"
>> + cmdsrpm = 'fakeroot ' + cmdsrpm + " -bs " + outspecfile
>> cmd = cmd + " -bb " + outspecfile
>>
>> + # Build the source rpm package !
>> + if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True) and 
>> d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() == 'SRPM':
>> + d.setVar('SBUILDSPEC', cmdsrpm + "\n")
>> + d.setVarFlag('SBUILDSPEC', 'func', '1')
>> + bb.build.exec_func('SBUILDSPEC', d)
>> +
>> # Build the rpm package!
>> d.setVar('BUILDSPEC', cmd + "\n")
>> d.setVarFlag('BUILDSPEC', 'func', '1')
>

Re: [PATCH 2/7] archiver.bbclass: New bbclass for archiving sources, patches, logs and scripts

[Xiaofeng Yan]

On 2012年03月14日 10:57, Saul Wold wrote:
> On 03/13/2012 05:52 AM, Xiaofeng Yan wrote:
>> From: Xiaofeng Yan<xiaofeng.yan@windriver.com>
>>
>> 1 Archive sources in ${S} in the different stage 
>> (do_unpack,do_patch,do_configure).
>> 2 Archive patches including series
>> 3 Archive logs including scripts (.bb and .inc files)
>> 4 dump environment resources which show all variable and
>> functions used to xxx.showdata.dump when running a task
>> 5 dump all content in 's' including patches to file xxx.diff.gz
>>
>> All archiving packages will be deployed to ${DEPLOY_DIR}/sources/
>>
>> [#YOCTO 1977]
>>
> Xiaofeng,
>
> I am making a first pass on this, there might be more, after I use it 
> to create archives.
>
> Sau!
>
>> Signed-off-by: Xiaofeng Yan<xiaofeng.yan@windriver.com>
>> ---
>> meta/classes/archiver.bbclass | 421 
>> +++++++++++++++++++++++++++++++++++++++++
>> 1 files changed, 421 insertions(+), 0 deletions(-)
>> create mode 100644 meta/classes/archiver.bbclass
>>
>> diff --git a/meta/classes/archiver.bbclass 
>> b/meta/classes/archiver.bbclass
>> new file mode 100644
>> index 0000000..95e870d
>> --- /dev/null
>> +++ b/meta/classes/archiver.bbclass
>> @@ -0,0 +1,421 @@
>> +# This file is used for archiving sources ,patches,and logs to tarball.
>> +# It also output building environment to xxx.dump.data and create 
>> xxx.diff.gz to record
>> +# all content in ${S} to a diff file.
>> +
>> +EXCLUDE_FROM ?= ".pc autom4te.cache"
>> +ARCHIVE_TYPE ?= "TAR SRPM"
>> +DISTRO ?= "poky"
>> +
>> +def parse_var(d,var):
>> + ''' parse variable like ${PV} in "require xxx_${PV}.inc" to a real 
>> value. for example, change "require xxx_${PV}.inc" to "require 
>> xxx_1.2.inc" '''
>> + import re
>> + pat = re.compile('.*\$({(.*)}).*')
>> + if '$' not in var and '/' not in var:
>> + return var
>> + else:
>> + if '/' in var:
>> + return [i for i in var.split('/') if i.endswith('.inc')][0]
>> + elif '$' in var:
>> + m = pat.match(var)
>> + patstr = '\$' + m.group(1)
>> + var_str = m.group(2)
>> + return re.sub(patstr,d.getVar(var_str,True),var)
>> + else:
>> + return var
>> +
>> +
>> +def get_bb_inc(d):
>> + '''create a directory "script-logs" including .bb and .inc file in 
>> ${WORKDIR}'''
>> + import re
>> + import os
>> + import shutil
>> +
>> + bbinc = []
>> + pat=re.compile('require\s*([^\s]*\.*)(.*)')
>> + file_dir = d.getVar('FILE', True)
>> + bbdir = os.path.dirname(file_dir)
>> + work_dir = d.getVar('WORKDIR', True)
>> + os.chdir(work_dir)
>> + bb.mkdirhier("script-logs")
>> + os.chdir(bbdir)
>> + bbfile = os.path.basename(file_dir)
>> + bbinc.append(bbfile)
>> +
>> + def get_inc (file):
>> + f = open(file,'r')
>> + for line in f.readlines():
>> + if 'require' not in line:
>> + bbinc.append(file)
>> + else:
>> + try:
>> + incfile = pat.match(line).group(1)
>> + incfile = parse_var(d,incfile)
>> + bbinc.append(incfile)
>> + get_inc(incfile)
>> + except AttributeError:
>> + pass
>> + get_inc(bbfile)
>> + os.chdir(work_dir)
>> + for root, dirs, files in os.walk(bbdir):
>> + for file in bbinc:
>> + if file in files:
>> + shutil.copy(root + '/' + file,'script-logs')
>> + oe.path.copytree('temp', 'script-logs')
>> + return work_dir + '/script-logs'
>> +
>> +def get_all_patches(d):
>> + '''copy patches and series file to a pointed directory which will 
>> be archived to tarball in ${WORKDIR}'''
>> + import shutil
>> +
>> + src_patches=[]
>> + pf = d.getVar('PF', True)
>> + work_dir = d.getVar('WORKDIR', True)
>> + dest = os.path.join(work_dir, pf + '-patches')
>> + shutil.rmtree(dest, ignore_errors=True)
>> + bb.mkdirhier(dest)
>> +
>> + src_uri = d.getVar('SRC_URI', 1).split()
>> + fetch = bb.fetch2.Fetch(src_uri, d)
>> + locals = (fetch.localpath(url) for url in fetch.urls)
>> + for local in locals:
>> + src_patches.append(local)
>> + for patch in src_patches[1:]:
>> + shutil.copy(patch,dest)
>> + return dest
>> +
>> +def get_applying_patches(d):
>> + """only copy applying patches to a pointed directory which will be 
>> archived to tarball"""
>> + import os
>> + import shutil
>> +
>> +
>> + pf = d.getVar('PF', True)
>> + work_dir = d.getVar('WORKDIR', True)
>> + dest = os.path.join(work_dir, pf + '-patches')
>> + shutil.rmtree(dest, ignore_errors=True)
>> + bb.mkdirhier(dest)
>> +
>> +
>> + patches = src_patches(d)
>> + for patch in patches:
>> + _, _, local, _, _, parm = bb.decodeurl(patch)
>> + if local:
>> + shutil.copy(local,dest)
>> + return dest
>> +
>> +def not_tarball(d):
>> + '''packages including key words 'work-shared','native', 'task-' 
>> will be passed'''
>> + import os
>> +
>> + workdir = d.getVar('WORKDIR',True)
>> + s = d.getVar('S',True)
>> + if 'work-shared' in s or 'task-' in workdir or 'native' in workdir:
>> + return True
>> + else:
>> + return False
>> +
>> +def archive_sources(d,middle_name):
>> + '''archive sources codes tree to tarball'''
>> + import tarfile
>> + import shutil
>> +
>> + s = d.getVar('S',True)
>> + workdir=d.getVar('WORKDIR', True)
>> + PF = d.getVar('PF',True)
>> + tarname = PF + '-' + middle_name + ".tar.gz"
>> +
>> + if os.path.exists(s) and s is not workdir:
>> + sourcedir = os.path.basename(s)
>> + tarbase = os.path.dirname(s)
>> + if not sourcedir or os.path.dirname(tarbase) == workdir:
>> + sourcedir = os.path.basename(os.path.dirname(s))
>> + tarbase = os.path.dirname(os.path.dirname(s))
>> + os.chdir(tarbase)
>> + else:
>> + sourcedir = os.path.basename(s)
>> + if not os.path.exists(sourcedir):
>> + os.mkdir(sourcedir)
>> + try:
>> + for file in os.listdir(s):
>> + if file is not 'temp' and file is not sourcedir:
>> + shutil.copy(file,sourcedir)
>> + except (IOError,OSError):
>> + pass
>> +
>> +
>> + if (len(os.listdir(sourcedir))) != 0:
>> + tar = tarfile.open( tarname, "w:gz")
>> + tar.add(sourcedir)
>> + tar.close()
>> + if cmp(workdir,os.path.dirname(s)) and not os.path.exists(workdir + 
>> '/' + tarname):
>> + shutil.move(os.path.dirname(s) + '/' + tarname,workdir)
>> + else:
>> + return
>> + return tarname
>> +
>> +def archive_patches(d,patchdir,series):
>> + '''archive patches to tarball and also include series files if 
>> 'series' is True'''
>> + import tarfile
>> + import shutil
>> +
>> + s = d.getVar('S',True)
>> + work_dir = d.getVar('WORKDIR', True)
>> + os.chdir(work_dir)
>> + patch_dir = os.path.basename(patchdir)
>> + tarname = patch_dir + ".tar.gz"
>> + if series == 'all' and os.path.exists(s + '/patches/series'):
>> + shutil.copy(s + '/patches/series',patch_dir)
>> + tar = tarfile.open(tarname, "w:gz")
>> + tar.add(patch_dir)
>> + tar.close()
>> + shutil.rmtree(patch_dir, ignore_errors=True)
>> + return tarname
>> +
>> +def select_archive_patches(d,option):
>> + '''select to archive all patches including non-applying and series 
>> or applying patches '''
>> + if option == "all":
>> + patchdir = get_all_patches(d)
>> + elif option == "applying":
>> + patchdir = get_applying_patches(d)
>> + try:
>> + os.rmdir(patchdir)
>> + except OSError:
>> + tarpatch = archive_patches(d,patchdir,option)
>> + return tarpatch
>> + return
>> +
>> +def archive_logs(d,logdir,bbinc=False):
>> + '''archive logs in temp to tarball and .bb and .inc files if bbinc 
>> is True '''
>> + import tarfile
>> + import shutil
>> +
>> + log_dir = os.path.basename(logdir)
>> + pf = d.getVar('PF',True)
>> + tarname = pf + '-' + log_dir + ".tar.gz"
>> + tar = tarfile.open(tarname, "w:gz")
>> + tar.add(log_dir)
>> + tar.close()
>> + if bbinc:
>> + shutil.rmtree(log_dir, ignore_errors=True)
>> + return tarname
>> +
>> +def get_licenses(d):
>> + '''get licenses for running .bb file'''
>> + licenses = d.getVar('LICENSE', 1).replace('&', '|')
>> + licenses = licenses.replace('(', '').replace(')', '')
>> + clean_licenses = ""
>> + for x in licenses.split():
>> + if x.strip() == '' or x == 'CLOSED':
>> + continue
>> + if x != "|":
>> + clean_licenses += x
>> + if '|' in clean_licenses:
>> + clean_licenses = clean_licenses.replace('|','')
>> + return clean_licenses
>> +
> This seems to be duplicated in your RPM code also? Move to 
> licenses.bbclass maybe?
I will export this function in archiver.bbclass and reference this here.
>
>> +
>> +def move_tarball_deploy(d,tarball_list):
>> + '''move tarball in location to ${DEPLOY_DIR}/sources'''
>> + import shutil
>> +
>> + if tarball_list is []:
>> + return
>> + target_sys = d.getVar('TARGET_SYS', True)
>> + pf = d.getVar('PF', True)
>> + licenses = get_licenses(d)
>> + tar_sources = d.getVar('DEPLOY_DIR', True) + '/sources/' + 
>> target_sys + '/' + licenses + '/' + pf
>> + if not os.path.exists(tar_sources):
>> + bb.mkdirhier(tar_sources)
>> + for source in tarball_list:
>> + if source:
>> + if os.path.exists(tar_sources + '/' + source):
>> + os.remove(tar_sources + '/' + source)
>> + shutil.move(source,tar_sources)
>> +
>> +def verify_var(d):
>> + '''check the type for archiving package('tar' or 'srpm')'''
>> + try:
>> + if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() not in 
>> d.getVar('ARCHIVE_TYPE', True).split():
>> + raise AttributeError
>> + except AttributeError:
>> + bb.fatal("\"SOURCE_ARCHIVE_PACKAGE_TYPE\" is \'tar\' or \'srpm\', 
>> no other types")
>> +
>> +def archive_copyleft(d):
>> + '''check if a package to archive is copy-left'''
>> + if bb.data.inherits_class('copyleft_compliance', d):
>> + included, reason = copyleft_should_include(d)
>> + if not included:
>> + return False
>> + return True
>> +def archive_sources_patches(d,middle_name):
>> + '''archive sources and patches to tarball. middle_name will append 
>> strings ${middle_name} to ${PR} as middle name. for example, 
>> zlib-1.4.6-prepatch(middle_name).tar.gz '''
>> + verify_var(d)
>> + if not_tarball(d):
>> + return
>> +
>> + source_tar_name = archive_sources(d,middle_name)
>> + if middle_name == "prepatch":
>> + if d.getVar('PATCHES_ARCHIVE_WITH_SERIES',True).upper() == 'TRUE':
>> + patch_tar_name = select_archive_patches(d,"all")
>> + elif d.getVar('PATCHES_ARCHIVE_WITH_SERIES',True).upper() == 'FALSE':
>> + patch_tar_name = select_archive_patches(d,"applying")
>> + else:
>> + bb.fatal("Please define 'PATCHES_ARCHIVE_WITH_SERIES' is strings 
>> 'True' or 'False' ")
>> + else:
>> + patch_tar_name = ''
>> +
>> + if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() not in 
>> 'SRPM':
>> + move_tarball_deploy(d,[source_tar_name,patch_tar_name])
>> +
>> +def archive_sources_patches_logs_copyleft(d,middle_name):
>> + '''archive source, patches and according to the variable 
>> "COPYLEFT_COMPLIANCE", If this variable is 'True', then archive the 
>> packages for copy-left, or else archive all packages'''
>> + verify_var(d)
>> + if not_tarball(d):
>> + return
>> + copyleft_compliance = d.getVar('COPYLEFT_COMPLIANCE', True)
>> + if copyleft_compliance is None:
>> + archive_sources_patches(d,middle_name)
>> + elif copyleft_compliance.upper() == 'TRUE' and archive_copyleft(d):
>> + archive_sources_patches(d,middle_name)
>> +
>> +
>> +def archive_scripts_logs(d):
>> + '''archive scripts and logs. scripts include .bb and .inc files and 
>> logs include stuff in "temp".'''
>> +
>> + s = d.getVar('WORKDIR', True)
>> + os.chdir(s)
>> + source_archive_log_with_scripts = 
>> d.getVar('SOURCE_ARCHIVE_LOG_WITH_SCRIPTS', True)
>> + if source_archive_log_with_scripts == 'logs_with_scripts':
>> + logdir = get_bb_inc(d)
>> + tarlog = archive_logs(d,logdir,True)
>> + elif source_archive_log_with_scripts == 'logs':
>> + if os.path.exists('temp'):
>> + archive_logs(d,'temp',False)
>> + else:
>> + return
>> +
>> + if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() not in 
>> 'SRPM':
>> + move_tarball_deploy(d,[tarlog])
>> +
>> +def archive_scripts_logs_copyleft(d):
>> + '''archive logs according to the variable "COPYLEFT_COMPLIANCE", If 
>> this variable is 'True', then archive the packages for copy-left, or 
>> else archive all packages'''
>> + verify_var(d)
>> + if not d.getVar('SOURCE_ARCHIVE_LOG_WITH_SCRIPTS', True) or 
>> not_tarball(d):
>> + return
>> + copyleft_compliance = d.getVar('COPYLEFT_COMPLIANCE', True)
>> + if copyleft_compliance is None:
>> + archive_scripts_logs(d)
>> + elif copyleft_compliance.upper() == 'TRUE' and archive_copyleft(d):
>> + archive_scripts_logs(d)
>> +
>> +
>> +def dumpdata(d):
>> + '''dump environment to "${P}-${PR}.showdata.dump" including all 
>> kinds of variables and functions when running a task'''
>> + workdir = bb.data.getVar('WORKDIR', d, 1)
>> + distro = bb.data.getVar('DISTRO', d, 1)
>> + s = d.getVar('S', True)
>> + pf = d.getVar('PF', True)
>> + target_sys = d.getVar('TARGET_SYS', True)
>> + licenses = get_licenses(d)
>> + dumpdir = d.getVar('DEPLOY_DIR', True) + '/sources/' + target_sys + 
>> '/' + licenses + '/' + pf
>> + if not os.path.exists(dumpdir):
>> + bb.mkdirhier(dumpdir)
>> +
>> + dumpfile = os.path.join(dumpdir, 
>> bb.data.expand("${P}-${PR}.showdata.dump",d))
>> +
>> + bb.note("Dumping metadata into '%s'" % dumpfile)
>> + f = open(dumpfile, "w")
>> + # emit variables and shell functions
>> + bb.data.emit_env(f, d, True)
>> + # emit the metadata which isnt valid shell
>> + for e in d.keys():
>> + if bb.data.getVarFlag(e, 'python', d):
>> + f.write("\npython %s () {\n%s}\n" % (e, bb.data.getVar(e, d, 1)))
>> + f.close()
>> +
>> +
>> +
>> +def create_diff_gz(d):
>> + '''creating .diff.gz in ${DEPLOY_DIR_SRC}/${P}-${PR}.diff.g gz for 
>> mapping all content in 's' including patches to xxx.diff.gz'''
>> + import shutil
>> +
>> + work_dir = d.getVar('WORKDIR', True)
>> + exclude_from = d.getVar('EXCLUDE_FROM', True).split()
>> + pf = d.getVar('PF', True)
>> + licenses = get_licenses(d)
>> + target_sys = d.getVar('TARGET_SYS', True)
>> + diff_dir = d.getVar('DEPLOY_DIR', True) + '/sources/' + target_sys 
>> + '/' + licenses + '/' + pf
>> + diff_file = os.path.join(diff_dir, 
>> bb.data.expand("${P}-${PR}.diff.gz",d))
>> + os.chdir(work_dir)
>> + f = open('temp/exclude-from-file', 'a')
>> + for i in exclude_from:
>> + f.write(i)
>> + f.write("\n")
>> + f.close()
>> +
>> +
>> + s=d.getVar('S', True)
>> + distro = d.getVar('DISTRO',True)
>> + dest = s + '/' + distro + '/files'
>> + if not os.path.exists(dest):
>> + bb.mkdirhier(dest)
>> + for i in os.listdir(os.getcwd()):
>> + if os.path.isfile(i):
>> + shutil.copy(i, dest)
>> +
>> + bb.note("Creating .diff.gz in ${DEPLOY_DIR_SRC}/${P}-${PR}.diff.gz")
>> + cmd = "LC_ALL=C TZ=UTC0 diff --exclude-from=" + work_dir + 
>> "/temp/exclude-from-file -Naur " + s + '.org' + ' ' + s + " | gzip 
>> -c> " + diff_file
>> + d.setVar('DIFF', cmd + "\n")
>> + d.setVarFlag('DIFF', 'func', '1')
>> + bb.build.exec_func('DIFF', d)
>> + shutil.rmtree(s + '.org', ignore_errors=True)
>> +
>> +# This function will run when user want to get tarball for sources 
>> and patches after do_unpack
>> +python do_archive_original_sources_patches(){
>> + archive_sources_patches_logs_copyleft(d,'prepatch')
>> +}
>> +
>> +# This function will run when user want to get tarball for patched 
>> sources after do_patch
>> +python do_archive_patched_sources(){
>> + archive_sources_patches_logs_copyleft(d,'patched')
>> +}
>> +
>> +# This function will run when user want to get tarball for 
>> configured sources after do_configure
>> +python do_archive_configured_sources(){
>> + archive_sources_patches_logs_copyleft(d,'configured')
>> +}
>> +
>> +# This function will run when user want to get tarball for logs or 
>> both logs and scripts(.bb and .inc files)
>> +python do_archive_scripts_logs_copyleft(){
>> + archive_scripts_logs_copyleft(d)
>> +}
>> +
>> +# This function will run when user want to know what variable and 
>> functions in a running task are and also can get a diff file including
>> +# all content a package should include.
>> +python do_dumpdata_create_diff_gz(){
>> + dumpdata(d)
>> + create_diff_gz(d)
>> +}
>> +
>> +# This functions prepare for archiving "linux-yocto" because this 
>> package create directory 's' before do_patch instead of after do_unpack.
>> +# This is special control for archiving linux-yocto only.
>> +python do_archive_linux_yocto(){
>> + s = d.getVar('S', True)
>> + if 'linux-yocto' in s:
>> + source_tar_name = archive_sources(d,'')
>> + if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() not in 
>> 'SRPM':
>> + move_tarball_deploy(d,[source_tar_name,''])
>> +}
>> +do_kernel_checkout[postfuncs] += "do_archive_linux_yocto "
>> +
>> +# remove tarball for sources, patches and logs after creating srpm.
>> +python do_remove_tarball(){
>> + if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() == 'SRPM':
>> + work_dir = d.getVar('WORKDIR', True)
>> + os.chdir(work_dir)
>> + for file in os.listdir(os.getcwd()):
>> + if '.tar.gz' in file:
>> + os.remove(file)
> As I mentioned in the RPM code, this might not be the best way to do 
> this, and it could break builds if you remove a tarball that's 
> expected to be there.
>
> Maybe store the tarballs in a ${WORKDIR}/.. directory (same level as 
> temp).
>
This is a problem.
I will use other method instead of this method.
>> +}
>> +do_remove_taball[deptask] = "do_archive_scripts_logs"
>> +do_package_write_rpm[postfuncs] += "do_remove_tarball "
>

[PATCH 1/7] package_rpm: Add srpm function to this bbclass

[Xiaofeng Yan]

From: Xiaofeng Yan <xiaofeng.yan@windriver.com>

Add a new function to archive source, patches and logs to a source rpm
package. Every source rpm package will be deployed to
${DEPLOY_DIR}/sources/deploy-srpm.

[YOCTO #1977]

Signed-off-by: Xiaofeng Yan <xiaofeng.yan@windriver.com>
---
 meta/classes/package_rpm.bbclass |   47 ++++++++++++++++++++++++++++++++++++++
 1 files changed, 47 insertions(+), 0 deletions(-)

diff --git a/meta/classes/package_rpm.bbclass b/meta/classes/package_rpm.bbclass
index 68313ec..b4ee5c0 100644
--- a/meta/classes/package_rpm.bbclass
+++ b/meta/classes/package_rpm.bbclass
@@ -6,6 +6,7 @@ RPM="rpm"
 RPMBUILD="rpmbuild"
 
 PKGWRITEDIRRPM = "${WORKDIR}/deploy-rpms"
+PKGWRITEDIRSRPM = "${DEPLOY_DIR}/sources/deploy-srpm"
 
 python package_rpm_fn () {
 	d.setVar('PKGFN', d.getVar('PKG'))
@@ -475,6 +476,31 @@ python write_specfile () {
 	import textwrap
 	import oe.packagedata
 
+	# append information for logs and patches to %prep
+	def add_prep(d,spec_files_bottom):
+		if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True) and d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() == 'SRPM':
+			spec_files_bottom.append('%%prep -n %s' % d.getVar('PN', True) )
+			spec_files_bottom.append('%s' % "echo \"include logs and patches, Please check them in SOURCES\"")
+			spec_files_bottom.append('')
+
+	# get the name of tarball for sources, patches and logs
+	def get_tarballs(d):
+		if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True) and d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() == 'SRPM':
+			return get_package(d)
+	
+	# append the name of tarball to key word 'SOURCE' in xxx.spec.
+	def tail_source(d,source_list=[],patch_list=None):
+		if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True) and d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() == 'SRPM':
+			source_number = 0
+			patch_number = 0
+			for source in source_list:
+				spec_preamble_top.append('Source' + str(source_number) + ': %s' % source)
+				source_number += 1
+			if patch_list:
+				for patch in patch_list:
+					print_deps(patch, "Patch" + str(patch_number), spec_preamble_top, d)
+					patch_number += 1
+
 	# We need a simple way to remove the MLPREFIX from the package name,
 	# and dependency information...
 	def strip_multilib(name, d):
@@ -786,6 +812,7 @@ python write_specfile () {
 		del localdata
 		bb.utils.unlockfile(lf)
 
+	add_prep(d,spec_files_bottom)
 	spec_preamble_top.append('Summary: %s' % srcsummary)
 	spec_preamble_top.append('Name: %s' % srcname)
 	spec_preamble_top.append('Version: %s' % srcversion)
@@ -796,6 +823,8 @@ python write_specfile () {
 	spec_preamble_top.append('Group: %s' % srcsection)
 	spec_preamble_top.append('Packager: %s' % srcmaintainer)
 	spec_preamble_top.append('URL: %s' % srchomepage)
+	source_list = get_tarballs(d)
+	tail_source(d,source_list,None)
 
 	# Replaces == Obsoletes && Provides
 	if srcrreplaces and srcrreplaces.strip() != "":
@@ -900,6 +929,15 @@ python write_specfile () {
 python do_package_rpm () {
 	import os
 
+	def creat_srpm_dir(d):
+		if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True) and d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() == 'SRPM':
+			clean_licenses = get_licenses(d)
+			pkgwritesrpmdir = bb.data.expand('${PKGWRITEDIRSRPM}/${PACKAGE_ARCH_EXTEND}', d)
+			pkgwritesrpmdir = pkgwritesrpmdir + '/' + clean_licenses
+			bb.mkdirhier(pkgwritesrpmdir)
+			os.chmod(pkgwritesrpmdir, 0755)
+			return pkgwritesrpmdir
+
 	# We need a simple way to remove the MLPREFIX from the package name,
 	# and dependency information...
 	def strip_multilib(name, d):
@@ -1015,8 +1053,17 @@ python do_package_rpm () {
 	cmd = cmd + " --define 'debug_package %{nil}'"
 	cmd = cmd + " --define '_rpmfc_magic_path " + magicfile + "'"
 	cmd = cmd + " --define '_tmppath " + workdir + "'"
+	if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True) and d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() == 'SRPM':
+		cmdsrpm = cmd + " --define '_sourcedir " + workdir + "' --define '_srcrpmdir " + creat_srpm_dir(d) + "'"
+		cmdsrpm = 'fakeroot ' + cmdsrpm + " -bs " + outspecfile
 	cmd = cmd + " -bb " + outspecfile
 
+	# Build the source rpm package !
+	if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True) and d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() == 'SRPM':
+		d.setVar('SBUILDSPEC', cmdsrpm + "\n")
+		d.setVarFlag('SBUILDSPEC', 'func', '1')
+		bb.build.exec_func('SBUILDSPEC', d)
+
 	# Build the rpm package!
 	d.setVar('BUILDSPEC', cmd + "\n")
 	d.setVarFlag('BUILDSPEC', 'func', '1')
-- 
1.7.0.4