* Please help to review the Yocto 1656: Recipe creation/import script
[not found] <4F17C7D9.10506@windriver.com>
@ 2012-01-19 9:33 ` Kang Kai
2012-01-19 11:57 ` Paul Eggleton
` (2 more replies)
0 siblings, 3 replies; 5+ messages in thread
From: Kang Kai @ 2012-01-19 9:33 UTC (permalink / raw)
To: Joshua Lock; +Cc: poky@yoctoproject.org
[-- Attachment #1: Type: text/plain, Size: 300 bytes --]
Hi Josh,
The attachment is the the design document V2 and implement.
The V2 document changes are:
1 remove patch_set from argument list
2 deb is not support right now
Any comment is welcome.
And I will on vacation until Jan 30, so my reply may not in time.
Thanks& Regards,
Kai
[-- Attachment #2: recipe_creation_bb.txt --]
[-- Type: text/plain, Size: 1027 bytes --]
Design document for Recipe creation/import script
* Purpose
The feature is from Yocto 1.2 Bug 1656. A script or similar system
that would allow you to give is an upstream URL, tarball, patch set,
package (SRPM, or debian style) and generate a recipe based on those
instructions.
Name it with bitbake-createbb, and integrate into the build system
to use the build system infrastructure that similiar with bitbake-runtask.
* Usage:
bitbake-createbb <SRC_URI>
* Design Flow
1 download the source package from the SRC_URI
2 get package name and version from package or tarball name.
3 unpack to /tmp/bitbake-createbb-pid
4 if tarball, get summary and description from freshmeat.org, README files and pkgconfig files;
otherwise get them from SRPM spec file or debian control file
5 get license, license file and checksum from LICENSE/COPYING files or freshmeat.org.
6 get dependent packages
|-- get them by parse configure files
|-- just parse spec file
8 remove downloaded files
[-- Attachment #3: bitbake-createbb --]
[-- Type: text/plain, Size: 19557 bytes --]
#!/usr/bin/env python
import os
import sys
import shutil
import re
import urllib
import sha, md5, hashlib
tmpdir = "/tmp/bitbake_createbb.%s" % os.getpid()
bin_file_name = ''
pkgname = ''
pkgversion = ''
summary = ''
description = ''
homepage = ''
license = []
license_files = {}
licenses = {}
depends = []
src_uri = ''
src_uri_md5sum = ''
src_uri_sha256sum = ''
inherits = []
# support 2 package types: tar and rpm
pkgtype = ""
def usage():
print "This is the bitbake import bb file script. Usage:\n"
print "\tbitbake-createbb <SRC_URI> \n"
def download(pkg_uri):
try:
shutil.rmtree(tmpdir)
except:
pass
cwd = os.getcwd()
os.mkdir(tmpdir)
os.chdir(tmpdir)
print 'Downloading package: ', pkg_uri, ' to ', tmpdir
ret = os.system("wget --quiet " + pkg_uri)
os.chdir(cwd)
return ret
def guess_name_by_uri(uri):
global bin_file_name, pkgname, pkgversion, pkgtype
elems = uri.rsplit('/', 1)
pkg = bin_file_name = elems[1]
if pkg.endswith("tgz"):
pkg = re.sub("tgz$", "tar.gz", pkg)
match = re.match('(.*?)\-([0-9\.\-\~]+)\.tar', pkg)
if match:
pkgtype = 'tar'
pkgname = match.group(1)
pkgversion = match.group(2)
match = re.match('(.*?)\-([0-9\.\~]+).*?\.src.rpm', pkg)
if match:
pkgtype = 'rpm'
pkgname = match.group(1)
pkgversion = match.group(2)
def get_md5_sha256_sum():
global src_uri_md5sum, src_uri_sha256sum
curdir = os.getcwd()
os.chdir(tmpdir)
f = file(bin_file_name, 'r')
content = f.read()
f.close()
md_five = md5.new()
md_five.update(content)
src_uri_md5sum = md_five.hexdigest()
sha_256 = hashlib.sha256()
sha_256.update(content)
src_uri_sha256sum = sha_256.hexdigest()
os.chdir(curdir)
def unpack_package():
global bin_file_name
cwd = os.getcwd()
os.chdir(tmpdir)
if pkgtype == 'rpm':
cmdline = "rpm2cpio " + bin_file_name + " | cpio -id"
ret = os.system(cmdline)
if ret != 0:
return ret
for item in os.listdir('.'):
if re.match(pkgname + '([0-9\.\-\~]*)\.tar', item):
bin_file_name = item
break
cmdline = "tar axf " + bin_file_name
ret = os.system(cmdline)
os.chdir(cwd)
return ret
# get the fall-back description when other way fail
# check homepage at same time
def guess_description_from_readme(readme):
global description, homepage
f = file(readme)
state = 0
desc = ''
for line in f:
if state == 1 and re.match('^\n', line) and len(desc) > 80:
state = 2
if state == 0 and len(line) > 1:
state = 1
if state == 1:
desc += line
match = re.search('(http\:\/\/.*$name.*\.org)', line)
if match:
url = match.group(1)
if re.search('bug') or len(homepage) > 1:
pass
else:
homepage = url
f.close()
if (len(desc) > 4 and len(description) < 3):
description = desc
def guess_description_from_freecode(pkgname):
global description
desc = ''
state = 0
html = urllib.urlopen("http://freecode.com/projects/" + pkgname)
for line in html:
if state == 1:
desc += line
if state == 0 and re.search('\<div class\=\"project-detail\"\>', line):
state = 1
if state == 1 and re.search('\<\/p\>', line):
state = 2
# deal the description
desc = re.sub('\<p\>', '', desc)
desc = re.sub('\<\/p\>', '', desc)
desc = re.sub('\r', '', desc)
desc = desc.strip()
if len(desc) > 10:
description = desc
# get Summary from pkgconfig file
def guess_summary_from_pc(pc):
global summary
fn = file(pc)
for line in fn:
match = re.match('Description:\s*(.*)', line)
if match and len(summary) < 2:
summary = match.group(1)
break
summary = summary.strip()
fn.close()
def guess_description(pkgdir):
global pkgname, description, summary
readmes = []
pcs = []
for subdir in os.listdir(pkgdir):
if re.match('^README$', subdir):
readmes.insert(0, os.path.join(pkgdir, subdir))
elif re.match('README.*', subdir):
readmes.append(os.path.join(pkgdir, subdir))
elif re.match('.*\.pc.*', subdir):
pcs.insert(0, os.path.join(pkgdir, subdir))
elif re.match(pkgname + '\.pc.*', subdir):
pcs.insert(0, os.path.join(pkgdir, subdir))
elif re.match('.*\.pc', subdir):
pcs.append(os.path.join(pkgdir, subdir))
for readme in readmes:
guess_description_from_readme(os.path.join(pkgdir, readme))
if (len(pkgname) > 2):
guess_description_from_freecode(pkgname)
for pc in pcs:
guess_summary_from_pc(pc)
# if didn't get summary, use first line of description
if len(summary) < 2:
summary = description
summary = re.sub("\n", " ", summary)
summary = re.sub("\s+", " ", summary)
match = re.match("(.*?)\.", summary)
summary = match.group(1)
# the sha1sum values are from autospectacle
def setup_licenses():
licenses['06877624ea5c77efe3b7e39b0f909eda6e25a4ec'] = "GPLv2"
licenses["075d599585584bb0e4b526f5c40cb6b17e0da35a"] = "GPLv2"
licenses["10782dd732f42f49918c839e8a5e2894c508b079"] = "GPLv2"
licenses["2d29c273fda30310211bbf6a24127d589be09b6c"] = "GPLv2"
licenses["4df5d4b947cf4e63e675729dd3f168ba844483c7"] = "LGPLv2.1"
licenses["503df7650052cf38efde55e85f0fe363e59b9739"] = "GPLv2"
licenses["5405311284eab5ab51113f87c9bfac435c695bb9"] = "GPLv2"
licenses["5fb362ef1680e635fe5fb212b55eef4db9ead48f"] = "LGPLv2"
licenses["68c94ffc34f8ad2d7bfae3f5a6b996409211c1b1"] = "GPLv2"
licenses["66c77efd1cf9c70d4f982ea59487b2eeb6338e26"] = "LGPLv2.1"
licenses["74a8a6531a42e124df07ab5599aad63870fa0bd4"] = "GPLv2"
licenses["8088b44375ef05202c0fca4e9e82d47591563609"] = "LGPLv2.1"
licenses["8624bcdae55baeef00cd11d5dfcfa60f68710a02"] = "GPLv3"
licenses["8e57ffebd0ed4417edc22e3f404ea3664d7fed27"] = "MIT"
licenses["99b5245b4714b9b89e7584bfc88da64e2d315b81"] = "BSD"
licenses["aba8d76d0af67d57da3c3c321caa59f3d242386b"] = "MPLv1.1"
licenses["bf50bac24e7ec325dbb09c6b6c4dcc88a7d79e8f"] = "LGPLv2"
licenses["caeb68c46fa36651acf592771d09de7937926bb3"] = "LGPLv2.1"
licenses["dfac199a7539a404407098a2541b9482279f690d"] = "GPLv2"
licenses["e60c2e780886f95df9c9ee36992b8edabec00bcc"] = "LGPLv2.1"
licenses["c931aad3017d975b7f20666cde0953234a9efde3"] = "GPLv2"
def guess_licenses_from_file(copying, relname):
global licenses, license
sha1 = sha.new()
fn = open(copying)
content = fn.read()
fn.close()
sha1.update(content)
digest = sha1.hexdigest()
if digest in licenses:
license.append(licenses[digest])
md_five = md5.new()
md_five.update(content)
license_files[relname] = md_five.hexdigest()
def guess_licenses_from_freecode():
global license
lic = ''
state = 0
html = urllib.urlopen("http://freecode.com/projects/" + pkgname)
for line in html:
if state == 1:
lic += line
if state == 0 and re.search('<th><span>Licenses</span></th>', line):
state = 1
if state == 1 and re.search('</a>', line):
state = 2
# deal the license
lic = lic.strip()
match = re.search('<a.*?>(.*?)</a>', lic)
if match:
license.append(match.group(1))
def guess_license(pkgdir):
for item in os.listdir(pkgdir):
realpath = os.path.join(pkgdir, item)
if os.path.isfile(realpath) and (re.match('COPY.*', item) \
or re.match('LICENSE.*', item) or re.match('GPL.*', item)):
guess_licenses_from_file(realpath, item)
if len(license) == 0:
guess_licenses_from_freecode()
def process_configure(pkgdir):
if os.path.isfile(os.path.join(pkgdir, 'autogen.sh')):
os.system("cd " + pkgdir + " ; ./autogen.sh &> /dev/null");
configure = os.path.join(pkgdir, 'configure')
if os.path.isfile(configure):
if 'autotools' not in inherits:
inherits.append('autotools')
fn = open(configure)
for line in fn:
match = re.match('PACKAGE_NAME=\'(.*?)\'', line)
if match and len(pkgname) == 0:
pkgname = match.group(1)
match = re.match('PACKAGE_TARNAME=\'(.*?)\'', line)
if match:
pkgname = match.group(1)
match = re.match('PACKAGE_VERSION=\'(.*?)\'', line)
if match:
pkgversion = match.group(1)
match = re.match('PACKAGE_URL=\'(.*?)\'', line)
if match:
homepage = match.group(1)
fn.close()
def push_buildreq(dep):
global depends
# remove collateral ] ) etc damage in the string
dep = re.sub("\"", "", dep)
dep = re.sub("\)", "", dep)
dep = re.sub("\]", "", dep)
dep = re.sub("\[", "", dep)
# first, undo the space packing
dep = re.sub(">=", " >= ", dep)
dep = re.sub("<=", " <= ", dep)
items = dep.split(' ')
dep = items[0]
# don't show configure variables, we can't deal with them
if re.search("\$", dep):
return
if re.search("AC_SUBST", dep):
return
if dep not in depends:
depends.append(dep)
def parse_configure_ac(ac_file):
depth = 0
clause = ""
fac = file(ac_file)
for line in fac:
line = line.strip()
i = 0
while i < len(line):
if line[i] == '(':
depth += 1
if line[i] == ')' and depth > 0:
depth -= 1
clause += line[i]
i += 1
if depth > 0:
continue
# remove '\n'
clause = re.sub('\n', '', clause)
clause = clause.strip()
match = re.match('PKG_CHECK_MODULES\((.*)\)', clause)
if match:
modules = match.group(1)
pkg = modules.split(',', 1)[1].strip()
match2 = re.match('\[(.*)\]', pkg)
if match2:
pkg = match2.group(1)
# split the build dependencies
pkg = pkg.replace('\s+', ' ')
pkg = re.sub('\s>\s', '>', pkg)
pkg = re.sub('\s>=\s', '>=', pkg)
pkg = re.sub('\s=\s', '=', pkg)
pkg = re.sub('\s<=\s', '<=', pkg)
pkg = re.sub('\s<\s', '<', pkg)
pkglist = pkg.split(' ')
for dep in pkglist:
push_buildreq(dep)
match = re.match('PKG_CHECK_EXISTS\((.*)\)', clause)
if match:
exists = match.group(1)
pkg = exists.split(',', 1)[0].strip()
match2 = re.match('\[(.*)\]', pkg)
if match2:
pkg = match2.group(1)
pkg = pkg.strip()
pkg = re.sub('\s+', ' ', pkg)
pkg = re.sub('\s>\s', '>', pkg)
pkg = re.sub('\s>=\s', '>=', pkg)
pkg = re.sub('\s=\s', '=', pkg)
pkg = re.sub('\s<=\s', '<=', pkg)
pkg = re.sub('\s<\s', '<', pkg)
pkglist = pkg.split(' ')
for dep in pkglist:
push_buildreq(dep)
# these items are from autospectacle.pl
if re.search('_PROG_INTLTOOL', clause):
push_buildreq("intltool")
if re.search('GETTEXT_PACKAGE', clause):
push_buildreq('gettext')
if re.search('GTK_DOC_CHECK', clause):
push_buildreq('gtk-doc')
if re.search('GNOME_DOC_INIT', clause):
push_buildreq('gnome-doc-utils')
if re.search('AM_GLIB_GNU_GETTEXT', clause):
push_buildreq('gettext')
match = re.search('AC_INIT\((.*)\)', clause)
if match:
ac_init = match.group(1)
ac_init = ac_init.strip()
ac_init = re.sub('\s+', ' ', ac_init)
version = ac_init.split(',')[1].strip()
if re.match('\d+(\.\d+)*', version):
pkgversion = version
match = re.search('AM_INIT_AUTOMAKE\((.*)\)', clause)
if match:
am_init = match.group(1)
am_init = re.sub('\s+', ' ', am_init)
items = am_init.split(',')
if len(items) >= 2:
ver = items[1]
ver = re.sub('\[', '', ver)
ver = re.sub('\]', '', ver)
if re.match('\d(\.\d+)*', ver):
pkgversion = ver
clause = ''
fac.close
def process_configure_ac(pkgdir):
configure_acs = []
for root, dirnames, filenames in os.walk(pkgdir):
for f in filenames:
if re.match('.*\.ac', f):
configure_acs.append(os.path.join(root, f))
if 'autotools' not in inherits:
inherits.append('autotools')
for ac in configure_acs:
parse_configure_ac(ac)
# check the *.pro files and get build requires from 'PKGCONFIG'
def process_qmake_pro(pkgdir):
global pkgname
pro_files = []
depth = 0
clause = ''
pre_char = ''
if os.path.isfile(os.path.join(pkgdir, pkgname + '.pro')) and 'qmake2' not in inherits:
inherits.append('qmake2')
for root, dirnames, filenames in os.walk(pkgdir):
for f in filenames:
if re.match('.*\.pro', f):
pro_files.append(os.path.join(pkgdir, f))
for pro in pro_files:
need_next_line = False
f = file(pro)
for line in f:
if line.startswith('#'):
continue
line = re.sub('\n', '', line)
clause += line
if line.endswith('\\'):
continue
clause = clause.strip()
match = re.match('PKGCONFIG\s*=(.*)', clause)
if match:
dep = match.group(1)
push_buildreq(dep)
match = re.match('PKGCONFIG\s*\+=(.*)', clause)
if match:
dep = match.group(1)
dep = dep.strip()
dep = re.sub('\\\\', '', dep)
dep = re.sub('\s+', ' ', dep)
items = dep.split(' ')
for item in items:
push_buildreq(item)
clause = ''
f.close()
def parse_spec_file(tmpdir):
# find the spec file first
spec_file = ''
for item in os.listdir(tmpdir):
if re.match('.*\.spec', item):
spec_file = os.path.join(tmpdir, item)
break
if len(spec_file) == 0:
return 1
global pkgname, pkgversion, summary, description, depends
global src_uri, homepage
f = file(spec_file)
for line in f:
match = re.match('Summary\s*:(.*)', line)
if match:
if len(summary) == 0:
summary = match.group(1).strip()
continue
match = re.match('Name\s*:(.*)', line)
if match:
pkgname = match.group(1).strip()
continue
match = re.match('Version\s*:(.*)', line)
if match:
version = match.group(1).strip()
if re.match('\d+(\.\d+)?', version):
pkgversion = version
continue
match = re.match('License\s*:(.*)', line)
if match:
license.append(match.group(1).strip())
continue
match = re.match('URL(?i)\s*:(.*)', line)
if match:
homepage = match.group(1).strip()
continue
match = re.match('Source0*\s*:(.*)', line)
if match:
src_uri = match.group(1).strip()
src_uri = re.sub('\%\{name\}', pkgname, src_uri)
src_uri = re.sub('\%\{version\}', pkgversion, src_uri)
continue
match = re.match('BuildRequires\s*:(.*)', line)
if match:
deps = match.group(1).strip()
dpes = re.sub(',', ' ', deps)
deps = re.sub('\s+', ' ', deps)
for item in deps.split(' '):
push_buildreq(item)
match = re.match('%description\s*$', line)
if match:
for l in f:
if l.startswith('%'):
break
description += l
description = re.sub('\n', '', description)
f.close()
def write_bbfile():
if len(pkgname) == 0 or len(pkgversion) == 0:
fail_quit("Can't get package name or version.")
content = ''
content += 'DESCRIPTION = "' + description + '"\n'
content += 'SUMMARY = "' + summary + '"\n'
if len(homepage) > 0:
content += 'HOMEPAGE = "' + homepage + '"\n'
content += 'LICENSE = "'
for lic in license:
content += lic + ' '
content += '"\n'
content += 'LIC_FILES_CHKSUM = "'
for key in license_files:
content += 'file://' + key + ';md5=' + license_files[key] + ' \\\n';
content += '\t\t"\n\n';
if len(depends) > 0:
content += 'DEPENDS = "'
for dep in depends:
content += dep + ' '
content += ' "\n'
content += 'PR = "r0"\n\n'
content += 'SRC_URI = "' + src_uri + '"\n'
content += 'SRC_URI[md5sum] = "' + src_uri_md5sum + '"\n'
content += 'SRC_URI[sha256sum] = "' + src_uri_sha256sum + '"\n'
if len(inherits) > 0:
content += '\ninherit '
for inherit in inherits:
content += inherit + ' '
content += '\n'
bb_filename = pkgname + '_' + pkgversion + '.bb'
bb_file = open(bb_filename, 'w')
bb_file.write(content)
bb_file.close()
print 'Create bb file : ', bb_filename
def fail_quit(msg):
print msg
clean_up()
exit(1)
def clean_up():
shutil.rmtree(tmpdir)
pass
##########################################################
if __name__ == '__main__':
if (len(sys.argv) < 2):
usage()
exit(1)
src_uri = sys.argv[1]
guess_name_by_uri(src_uri)
if pkgname is None:
print "Can't get the package name."
exit(1)
ret = download(src_uri)
if ret != 0:
fail_quit('Download package failed. Make sure the SRC_URI is valid.')
if unpack_package() != 0:
fail_quit('Unpack package failed.')
if pkgtype == 'rpm':
parse_spec_file(tmpdir)
get_md5_sha256_sum()
pkgdir = ''
for subdir in os.listdir(tmpdir):
subdir = os.path.join(tmpdir, subdir)
if os.path.isdir(subdir):
pkgdir = subdir
break
if pkgtype == 'tar':
parse_spec_file(pkgdir)
guess_description(pkgdir)
process_configure(pkgdir)
setup_licenses()
guess_license(pkgdir)
process_configure_ac(pkgdir)
process_qmake_pro(pkgdir)
write_bbfile()
clean_up()
^ permalink raw reply [flat|nested] 5+ messages in thread
* Re: Please help to review the Yocto 1656: Recipe creation/import script
2012-01-19 9:33 ` Please help to review the Yocto 1656: Recipe creation/import script Kang Kai
@ 2012-01-19 11:57 ` Paul Eggleton
2012-01-19 16:33 ` Mark Hatle
2012-01-20 1:57 ` Joshua Lock
2 siblings, 0 replies; 5+ messages in thread
From: Paul Eggleton @ 2012-01-19 11:57 UTC (permalink / raw)
To: Kang Kai; +Cc: poky
Hi Kai,
On Thursday 19 January 2012 17:33:47 Kang Kai wrote:
> The attachment is the the design document V2 and implement.
>
> The V2 document changes are:
> 1 remove patch_set from argument list
> 2 deb is not support right now
>
> Any comment is welcome.
> And I will on vacation until Jan 30, so my reply may not in time.
Great work! I can see this being really useful to help people get started with
new recipes. A few comments:
1) I tried it on hdparm and it worked very well. However, obexftp caused an
error:
----------------
Downloading package:
http://downloads.sourceforge.net/openobex/obexftp-0.23.tar.bz2 to
/tmp/bitbake_createbb.5729
Traceback (most recent call last):
File "./bitbake-createbb", line 643, in <module>
process_configure(pkgdir)
File "./bitbake-createbb", line 295, in process_configure
if match and len(pkgname) == 0:
UnboundLocalError: local variable 'pkgname' referenced before assignment
----------------
2) If it is able to find license file(s) but unable to find any appropriate for
LIC_FILES_CHKSUM, it would be nice if it would print a warning just so the
user knows they will need to set this themselves (bonus points if you still
list LIC_FILES_CHKSUM entries for license files that did not match the known
checksums either in a comment in the output or in the warning message, to save
the user work if they know the files are appropriate). If you prefer I can send
a patch afterwards for this.
3) If you have an RPM spec file it is possible to extract PACKAGES and FILES
and possibly even EXTRA_OECONF. Somewhere around here I have a very basic
script to do this I think, I could send it to you if you like (or we can leave
this for later).
Cheers,
Paul
--
Paul Eggleton
Intel Open Source Technology Centre
^ permalink raw reply [flat|nested] 5+ messages in thread
* Re: Please help to review the Yocto 1656: Recipe creation/import script
2012-01-19 9:33 ` Please help to review the Yocto 1656: Recipe creation/import script Kang Kai
2012-01-19 11:57 ` Paul Eggleton
@ 2012-01-19 16:33 ` Mark Hatle
2012-01-30 2:21 ` Kang Kai
2012-01-20 1:57 ` Joshua Lock
2 siblings, 1 reply; 5+ messages in thread
From: Mark Hatle @ 2012-01-19 16:33 UTC (permalink / raw)
To: poky
On 1/19/12 3:33 AM, Kang Kai wrote:
> Hi Josh,
>
> The attachment is the the design document V2 and implement.
>
> The V2 document changes are:
> 1 remove patch_set from argument list
> 2 deb is not support right now
>
> Any comment is welcome.
> And I will on vacation until Jan 30, so my reply may not in time.
>
> Thanks& Regards,
> Kai
>
>
>
>
> recipe_creation_bb.txt
>
>
> Design document for Recipe creation/import script
>
> * Purpose
>
> The feature is from Yocto 1.2 Bug 1656. A script or similar system
> that would allow you to give is an upstream URL, tarball, patch set,
> package (SRPM, or debian style) and generate a recipe based on those
> instructions.
>
> Name it with bitbake-createbb, and integrate into the build system
> to use the build system infrastructure that similiar with bitbake-runtask.
I'd suggest changing the name to recipe-createbb or something. Since this isn't
a "bitbake" program (or is it?)
> * Usage:
> bitbake-createbb<SRC_URI>
>
> * Design Flow
> 1 download the source package from the SRC_URI
> 2 get package name and version from package or tarball name.
> 3 unpack to /tmp/bitbake-createbb-pid
> 4 if tarball, get summary and description from freshmeat.org, README files and pkgconfig files;
> otherwise get them from SRPM spec file or debian control file
> 5 get license, license file and checksum from LICENSE/COPYING files or freshmeat.org.
For license files, I would suggest we scan for common filenames:
LICENSE, COPYING, COPYING.LIB, and COPYRIGHT
If it doesn't match the known checksums, we tell the user and let them manually
figure out the license type and verify things are consisten, but we should be
able to fill out the license filename for them.
Otherwise, it all looks fine to me.
--Mark
> 6 get dependent packages
> |-- get them by parse configure files
> |-- just parse spec file
> 8 remove downloaded files
>
>
>
>
>
> bitbake-createbb
>
>
> #!/usr/bin/env python
>
> import os
> import sys
> import shutil
> import re
> import urllib
> import sha, md5, hashlib
>
> tmpdir = "/tmp/bitbake_createbb.%s" % os.getpid()
> bin_file_name = ''
> pkgname = ''
> pkgversion = ''
> summary = ''
> description = ''
> homepage = ''
> license = []
> license_files = {}
> licenses = {}
> depends = []
> src_uri = ''
> src_uri_md5sum = ''
> src_uri_sha256sum = ''
> inherits = []
>
> # support 2 package types: tar and rpm
> pkgtype = ""
>
> def usage():
> print "This is the bitbake import bb file script. Usage:\n"
> print "\tbitbake-createbb<SRC_URI> \n"
>
> def download(pkg_uri):
> try:
> shutil.rmtree(tmpdir)
> except:
> pass
>
> cwd = os.getcwd()
> os.mkdir(tmpdir)
> os.chdir(tmpdir)
> print 'Downloading package: ', pkg_uri, ' to ', tmpdir
> ret = os.system("wget --quiet " + pkg_uri)
> os.chdir(cwd)
>
> return ret
>
> def guess_name_by_uri(uri):
> global bin_file_name, pkgname, pkgversion, pkgtype
>
> elems = uri.rsplit('/', 1)
> pkg = bin_file_name = elems[1]
>
> if pkg.endswith("tgz"):
> pkg = re.sub("tgz$", "tar.gz", pkg)
>
> match = re.match('(.*?)\-([0-9\.\-\~]+)\.tar', pkg)
> if match:
> pkgtype = 'tar'
> pkgname = match.group(1)
> pkgversion = match.group(2)
>
> match = re.match('(.*?)\-([0-9\.\~]+).*?\.src.rpm', pkg)
> if match:
> pkgtype = 'rpm'
> pkgname = match.group(1)
> pkgversion = match.group(2)
>
> def get_md5_sha256_sum():
> global src_uri_md5sum, src_uri_sha256sum
>
> curdir = os.getcwd()
> os.chdir(tmpdir)
>
> f = file(bin_file_name, 'r')
> content = f.read()
> f.close()
>
> md_five = md5.new()
> md_five.update(content)
> src_uri_md5sum = md_five.hexdigest()
>
> sha_256 = hashlib.sha256()
> sha_256.update(content)
> src_uri_sha256sum = sha_256.hexdigest()
>
> os.chdir(curdir)
>
> def unpack_package():
> global bin_file_name
>
> cwd = os.getcwd()
> os.chdir(tmpdir)
> if pkgtype == 'rpm':
> cmdline = "rpm2cpio " + bin_file_name + " | cpio -id"
> ret = os.system(cmdline)
> if ret != 0:
> return ret
> for item in os.listdir('.'):
> if re.match(pkgname + '([0-9\.\-\~]*)\.tar', item):
> bin_file_name = item
> break
>
> cmdline = "tar axf " + bin_file_name
> ret = os.system(cmdline)
>
> os.chdir(cwd)
> return ret
>
> # get the fall-back description when other way fail
> # check homepage at same time
> def guess_description_from_readme(readme):
> global description, homepage
> f = file(readme)
> state = 0
> desc = ''
>
> for line in f:
> if state == 1 and re.match('^\n', line) and len(desc)> 80:
> state = 2
> if state == 0 and len(line)> 1:
> state = 1
> if state == 1:
> desc += line
>
> match = re.search('(http\:\/\/.*$name.*\.org)', line)
> if match:
> url = match.group(1)
> if re.search('bug') or len(homepage)> 1:
> pass
> else:
> homepage = url
>
> f.close()
> if (len(desc)> 4 and len(description)< 3):
> description = desc
>
> def guess_description_from_freecode(pkgname):
> global description
> desc = ''
> state = 0
>
> html = urllib.urlopen("http://freecode.com/projects/" + pkgname)
> for line in html:
> if state == 1:
> desc += line
> if state == 0 and re.search('\<div class\=\"project-detail\"\>', line):
> state = 1
> if state == 1 and re.search('\<\/p\>', line):
> state = 2
>
> # deal the description
> desc = re.sub('\<p\>', '', desc)
> desc = re.sub('\<\/p\>', '', desc)
> desc = re.sub('\r', '', desc)
> desc = desc.strip()
> if len(desc)> 10:
> description = desc
>
> # get Summary from pkgconfig file
> def guess_summary_from_pc(pc):
> global summary
>
> fn = file(pc)
> for line in fn:
> match = re.match('Description:\s*(.*)', line)
> if match and len(summary)< 2:
> summary = match.group(1)
> break
>
> summary = summary.strip()
> fn.close()
>
> def guess_description(pkgdir):
> global pkgname, description, summary
>
> readmes = []
> pcs = []
>
> for subdir in os.listdir(pkgdir):
> if re.match('^README$', subdir):
> readmes.insert(0, os.path.join(pkgdir, subdir))
> elif re.match('README.*', subdir):
> readmes.append(os.path.join(pkgdir, subdir))
> elif re.match('.*\.pc.*', subdir):
> pcs.insert(0, os.path.join(pkgdir, subdir))
> elif re.match(pkgname + '\.pc.*', subdir):
> pcs.insert(0, os.path.join(pkgdir, subdir))
> elif re.match('.*\.pc', subdir):
> pcs.append(os.path.join(pkgdir, subdir))
>
> for readme in readmes:
> guess_description_from_readme(os.path.join(pkgdir, readme))
>
> if (len(pkgname)> 2):
> guess_description_from_freecode(pkgname)
>
> for pc in pcs:
> guess_summary_from_pc(pc)
>
> # if didn't get summary, use first line of description
> if len(summary)< 2:
> summary = description
> summary = re.sub("\n", " ", summary)
> summary = re.sub("\s+", " ", summary)
> match = re.match("(.*?)\.", summary)
> summary = match.group(1)
>
> # the sha1sum values are from autospectacle
> def setup_licenses():
> licenses['06877624ea5c77efe3b7e39b0f909eda6e25a4ec'] = "GPLv2"
> licenses["075d599585584bb0e4b526f5c40cb6b17e0da35a"] = "GPLv2"
> licenses["10782dd732f42f49918c839e8a5e2894c508b079"] = "GPLv2"
> licenses["2d29c273fda30310211bbf6a24127d589be09b6c"] = "GPLv2"
> licenses["4df5d4b947cf4e63e675729dd3f168ba844483c7"] = "LGPLv2.1"
> licenses["503df7650052cf38efde55e85f0fe363e59b9739"] = "GPLv2"
> licenses["5405311284eab5ab51113f87c9bfac435c695bb9"] = "GPLv2"
> licenses["5fb362ef1680e635fe5fb212b55eef4db9ead48f"] = "LGPLv2"
> licenses["68c94ffc34f8ad2d7bfae3f5a6b996409211c1b1"] = "GPLv2"
> licenses["66c77efd1cf9c70d4f982ea59487b2eeb6338e26"] = "LGPLv2.1"
> licenses["74a8a6531a42e124df07ab5599aad63870fa0bd4"] = "GPLv2"
> licenses["8088b44375ef05202c0fca4e9e82d47591563609"] = "LGPLv2.1"
> licenses["8624bcdae55baeef00cd11d5dfcfa60f68710a02"] = "GPLv3"
> licenses["8e57ffebd0ed4417edc22e3f404ea3664d7fed27"] = "MIT"
> licenses["99b5245b4714b9b89e7584bfc88da64e2d315b81"] = "BSD"
> licenses["aba8d76d0af67d57da3c3c321caa59f3d242386b"] = "MPLv1.1"
> licenses["bf50bac24e7ec325dbb09c6b6c4dcc88a7d79e8f"] = "LGPLv2"
> licenses["caeb68c46fa36651acf592771d09de7937926bb3"] = "LGPLv2.1"
> licenses["dfac199a7539a404407098a2541b9482279f690d"] = "GPLv2"
> licenses["e60c2e780886f95df9c9ee36992b8edabec00bcc"] = "LGPLv2.1"
> licenses["c931aad3017d975b7f20666cde0953234a9efde3"] = "GPLv2"
>
> def guess_licenses_from_file(copying, relname):
> global licenses, license
>
> sha1 = sha.new()
>
> fn = open(copying)
> content = fn.read()
> fn.close()
>
> sha1.update(content)
> digest = sha1.hexdigest()
>
> if digest in licenses:
> license.append(licenses[digest])
> md_five = md5.new()
> md_five.update(content)
> license_files[relname] = md_five.hexdigest()
>
> def guess_licenses_from_freecode():
> global license
>
> lic = ''
> state = 0
>
> html = urllib.urlopen("http://freecode.com/projects/" + pkgname)
> for line in html:
> if state == 1:
> lic += line
> if state == 0 and re.search('<th><span>Licenses</span></th>', line):
> state = 1
> if state == 1 and re.search('</a>', line):
> state = 2
>
> # deal the license
> lic = lic.strip()
> match = re.search('<a.*?>(.*?)</a>', lic)
> if match:
> license.append(match.group(1))
>
> def guess_license(pkgdir):
> for item in os.listdir(pkgdir):
> realpath = os.path.join(pkgdir, item)
> if os.path.isfile(realpath) and (re.match('COPY.*', item) \
> or re.match('LICENSE.*', item) or re.match('GPL.*', item)):
> guess_licenses_from_file(realpath, item)
>
> if len(license) == 0:
> guess_licenses_from_freecode()
>
> def process_configure(pkgdir):
> if os.path.isfile(os.path.join(pkgdir, 'autogen.sh')):
> os.system("cd " + pkgdir + " ; ./autogen.sh&> /dev/null");
>
> configure = os.path.join(pkgdir, 'configure')
> if os.path.isfile(configure):
> if 'autotools' not in inherits:
> inherits.append('autotools')
>
> fn = open(configure)
> for line in fn:
> match = re.match('PACKAGE_NAME=\'(.*?)\'', line)
> if match and len(pkgname) == 0:
> pkgname = match.group(1)
>
> match = re.match('PACKAGE_TARNAME=\'(.*?)\'', line)
> if match:
> pkgname = match.group(1)
>
> match = re.match('PACKAGE_VERSION=\'(.*?)\'', line)
> if match:
> pkgversion = match.group(1)
>
> match = re.match('PACKAGE_URL=\'(.*?)\'', line)
> if match:
> homepage = match.group(1)
>
> fn.close()
>
> def push_buildreq(dep):
> global depends
>
> # remove collateral ] ) etc damage in the string
> dep = re.sub("\"", "", dep)
> dep = re.sub("\)", "", dep)
> dep = re.sub("\]", "", dep)
> dep = re.sub("\[", "", dep)
>
> # first, undo the space packing
> dep = re.sub(">=", ">= ", dep)
> dep = re.sub("<=", "<= ", dep)
>
> items = dep.split(' ')
> dep = items[0]
>
> # don't show configure variables, we can't deal with them
> if re.search("\$", dep):
> return
> if re.search("AC_SUBST", dep):
> return
>
> if dep not in depends:
> depends.append(dep)
>
> def parse_configure_ac(ac_file):
> depth = 0
> clause = ""
>
> fac = file(ac_file)
>
> for line in fac:
> line = line.strip()
> i = 0
> while i< len(line):
> if line[i] == '(':
> depth += 1
> if line[i] == ')' and depth> 0:
> depth -= 1
> clause += line[i]
> i += 1
> if depth> 0:
> continue
>
> # remove '\n'
> clause = re.sub('\n', '', clause)
> clause = clause.strip()
>
> match = re.match('PKG_CHECK_MODULES\((.*)\)', clause)
> if match:
> modules = match.group(1)
> pkg = modules.split(',', 1)[1].strip()
> match2 = re.match('\[(.*)\]', pkg)
> if match2:
> pkg = match2.group(1)
>
> # split the build dependencies
> pkg = pkg.replace('\s+', ' ')
> pkg = re.sub('\s>\s', '>', pkg)
> pkg = re.sub('\s>=\s', '>=', pkg)
> pkg = re.sub('\s=\s', '=', pkg)
> pkg = re.sub('\s<=\s', '<=', pkg)
> pkg = re.sub('\s<\s', '<', pkg)
> pkglist = pkg.split(' ')
> for dep in pkglist:
> push_buildreq(dep)
>
> match = re.match('PKG_CHECK_EXISTS\((.*)\)', clause)
> if match:
> exists = match.group(1)
> pkg = exists.split(',', 1)[0].strip()
> match2 = re.match('\[(.*)\]', pkg)
> if match2:
> pkg = match2.group(1)
> pkg = pkg.strip()
>
> pkg = re.sub('\s+', ' ', pkg)
> pkg = re.sub('\s>\s', '>', pkg)
> pkg = re.sub('\s>=\s', '>=', pkg)
> pkg = re.sub('\s=\s', '=', pkg)
> pkg = re.sub('\s<=\s', '<=', pkg)
> pkg = re.sub('\s<\s', '<', pkg)
> pkglist = pkg.split(' ')
> for dep in pkglist:
> push_buildreq(dep)
>
> # these items are from autospectacle.pl
> if re.search('_PROG_INTLTOOL', clause):
> push_buildreq("intltool")
> if re.search('GETTEXT_PACKAGE', clause):
> push_buildreq('gettext')
> if re.search('GTK_DOC_CHECK', clause):
> push_buildreq('gtk-doc')
> if re.search('GNOME_DOC_INIT', clause):
> push_buildreq('gnome-doc-utils')
> if re.search('AM_GLIB_GNU_GETTEXT', clause):
> push_buildreq('gettext')
>
> match = re.search('AC_INIT\((.*)\)', clause)
> if match:
> ac_init = match.group(1)
> ac_init = ac_init.strip()
> ac_init = re.sub('\s+', ' ', ac_init)
> version = ac_init.split(',')[1].strip()
> if re.match('\d+(\.\d+)*', version):
> pkgversion = version
>
> match = re.search('AM_INIT_AUTOMAKE\((.*)\)', clause)
> if match:
> am_init = match.group(1)
> am_init = re.sub('\s+', ' ', am_init)
> items = am_init.split(',')
> if len(items)>= 2:
> ver = items[1]
> ver = re.sub('\[', '', ver)
> ver = re.sub('\]', '', ver)
> if re.match('\d(\.\d+)*', ver):
> pkgversion = ver
> clause = ''
> fac.close
>
> def process_configure_ac(pkgdir):
> configure_acs = []
> for root, dirnames, filenames in os.walk(pkgdir):
> for f in filenames:
> if re.match('.*\.ac', f):
> configure_acs.append(os.path.join(root, f))
> if 'autotools' not in inherits:
> inherits.append('autotools')
>
> for ac in configure_acs:
> parse_configure_ac(ac)
>
> # check the *.pro files and get build requires from 'PKGCONFIG'
> def process_qmake_pro(pkgdir):
> global pkgname
>
> pro_files = []
> depth = 0
> clause = ''
> pre_char = ''
>
> if os.path.isfile(os.path.join(pkgdir, pkgname + '.pro')) and 'qmake2' not in inherits:
> inherits.append('qmake2')
> for root, dirnames, filenames in os.walk(pkgdir):
> for f in filenames:
> if re.match('.*\.pro', f):
> pro_files.append(os.path.join(pkgdir, f))
>
> for pro in pro_files:
> need_next_line = False
> f = file(pro)
> for line in f:
> if line.startswith('#'):
> continue
> line = re.sub('\n', '', line)
> clause += line
> if line.endswith('\\'):
> continue
>
> clause = clause.strip()
> match = re.match('PKGCONFIG\s*=(.*)', clause)
> if match:
> dep = match.group(1)
> push_buildreq(dep)
> match = re.match('PKGCONFIG\s*\+=(.*)', clause)
> if match:
> dep = match.group(1)
> dep = dep.strip()
> dep = re.sub('\\\\', '', dep)
> dep = re.sub('\s+', ' ', dep)
> items = dep.split(' ')
> for item in items:
> push_buildreq(item)
>
> clause = ''
>
> f.close()
>
> def parse_spec_file(tmpdir):
> # find the spec file first
> spec_file = ''
> for item in os.listdir(tmpdir):
> if re.match('.*\.spec', item):
> spec_file = os.path.join(tmpdir, item)
> break
>
> if len(spec_file) == 0:
> return 1
>
> global pkgname, pkgversion, summary, description, depends
> global src_uri, homepage
>
> f = file(spec_file)
> for line in f:
> match = re.match('Summary\s*:(.*)', line)
> if match:
> if len(summary) == 0:
> summary = match.group(1).strip()
> continue
> match = re.match('Name\s*:(.*)', line)
> if match:
> pkgname = match.group(1).strip()
> continue
> match = re.match('Version\s*:(.*)', line)
> if match:
> version = match.group(1).strip()
> if re.match('\d+(\.\d+)?', version):
> pkgversion = version
> continue
> match = re.match('License\s*:(.*)', line)
> if match:
> license.append(match.group(1).strip())
> continue
> match = re.match('URL(?i)\s*:(.*)', line)
> if match:
> homepage = match.group(1).strip()
> continue
> match = re.match('Source0*\s*:(.*)', line)
> if match:
> src_uri = match.group(1).strip()
> src_uri = re.sub('\%\{name\}', pkgname, src_uri)
> src_uri = re.sub('\%\{version\}', pkgversion, src_uri)
> continue
> match = re.match('BuildRequires\s*:(.*)', line)
> if match:
> deps = match.group(1).strip()
> dpes = re.sub(',', ' ', deps)
> deps = re.sub('\s+', ' ', deps)
> for item in deps.split(' '):
> push_buildreq(item)
>
> match = re.match('%description\s*$', line)
> if match:
> for l in f:
> if l.startswith('%'):
> break
> description += l
> description = re.sub('\n', '', description)
>
> f.close()
>
> def write_bbfile():
> if len(pkgname) == 0 or len(pkgversion) == 0:
> fail_quit("Can't get package name or version.")
>
> content = ''
> content += 'DESCRIPTION = "' + description + '"\n'
> content += 'SUMMARY = "' + summary + '"\n'
> if len(homepage)> 0:
> content += 'HOMEPAGE = "' + homepage + '"\n'
>
> content += 'LICENSE = "'
> for lic in license:
> content += lic + ' '
> content += '"\n'
> content += 'LIC_FILES_CHKSUM = "'
> for key in license_files:
> content += 'file://' + key + ';md5=' + license_files[key] + ' \\\n';
> content += '\t\t"\n\n';
>
> if len(depends)> 0:
> content += 'DEPENDS = "'
> for dep in depends:
> content += dep + ' '
> content += ' "\n'
>
> content += 'PR = "r0"\n\n'
>
> content += 'SRC_URI = "' + src_uri + '"\n'
> content += 'SRC_URI[md5sum] = "' + src_uri_md5sum + '"\n'
> content += 'SRC_URI[sha256sum] = "' + src_uri_sha256sum + '"\n'
>
> if len(inherits)> 0:
> content += '\ninherit '
> for inherit in inherits:
> content += inherit + ' '
> content += '\n'
>
> bb_filename = pkgname + '_' + pkgversion + '.bb'
> bb_file = open(bb_filename, 'w')
> bb_file.write(content)
> bb_file.close()
> print 'Create bb file : ', bb_filename
>
> def fail_quit(msg):
> print msg
> clean_up()
> exit(1)
>
> def clean_up():
> shutil.rmtree(tmpdir)
> pass
>
> ##########################################################
>
> if __name__ == '__main__':
> if (len(sys.argv)< 2):
> usage()
> exit(1)
>
> src_uri = sys.argv[1]
>
> guess_name_by_uri(src_uri)
> if pkgname is None:
> print "Can't get the package name."
> exit(1)
>
> ret = download(src_uri)
> if ret != 0:
> fail_quit('Download package failed. Make sure the SRC_URI is valid.')
>
> if unpack_package() != 0:
> fail_quit('Unpack package failed.')
>
> if pkgtype == 'rpm':
> parse_spec_file(tmpdir)
>
> get_md5_sha256_sum()
>
> pkgdir = ''
> for subdir in os.listdir(tmpdir):
> subdir = os.path.join(tmpdir, subdir)
> if os.path.isdir(subdir):
> pkgdir = subdir
> break
>
> if pkgtype == 'tar':
> parse_spec_file(pkgdir)
>
> guess_description(pkgdir)
> process_configure(pkgdir)
>
> setup_licenses()
> guess_license(pkgdir)
>
> process_configure_ac(pkgdir)
> process_qmake_pro(pkgdir)
>
> write_bbfile()
> clean_up()
>
>
>
>
> _______________________________________________
> poky mailing list
> poky@yoctoproject.org
> https://lists.yoctoproject.org/listinfo/poky
>
^ permalink raw reply [flat|nested] 5+ messages in thread
* Re: Please help to review the Yocto 1656: Recipe creation/import script
2012-01-19 9:33 ` Please help to review the Yocto 1656: Recipe creation/import script Kang Kai
2012-01-19 11:57 ` Paul Eggleton
2012-01-19 16:33 ` Mark Hatle
@ 2012-01-20 1:57 ` Joshua Lock
2 siblings, 0 replies; 5+ messages in thread
From: Joshua Lock @ 2012-01-20 1:57 UTC (permalink / raw)
To: Kang Kai; +Cc: poky@yoctoproject.org
On 19/01/12 01:33, Kang Kai wrote:
> Hi Josh,
>
> The attachment is the the design document V2 and implement.
>
> The V2 document changes are:
> 1 remove patch_set from argument list
> 2 deb is not support right now
>
> Any comment is welcome.
> And I will on vacation until Jan 30, so my reply may not in time.
This looks like a good start, Paul has offered some great feedback. In
addition I'd add that we need a license header on the script, we should
probably attribute autospectacle in the header too (Inspired by? Based on?).
I haven't had chance to do a thorough code review but will try and do so
whilst you're on your break.
Cheers,
Joshua
--
Joshua Lock
Yocto Project "Johannes factotum"
Intel Open Source Technology Centre
^ permalink raw reply [flat|nested] 5+ messages in thread
* Re: Please help to review the Yocto 1656: Recipe creation/import script
2012-01-19 16:33 ` Mark Hatle
@ 2012-01-30 2:21 ` Kang Kai
0 siblings, 0 replies; 5+ messages in thread
From: Kang Kai @ 2012-01-30 2:21 UTC (permalink / raw)
To: Mark Hatle; +Cc: poky
On 2012年01月20日 00:33, Mark Hatle wrote:
> On 1/19/12 3:33 AM, Kang Kai wrote:
>> Hi Josh,
>>
>> The attachment is the the design document V2 and implement.
>>
>> The V2 document changes are:
>> 1 remove patch_set from argument list
>> 2 deb is not support right now
>>
>> Any comment is welcome.
>> And I will on vacation until Jan 30, so my reply may not in time.
>>
>> Thanks& Regards,
>> Kai
>>
>>
>>
>>
>> recipe_creation_bb.txt
>>
>>
>> Design document for Recipe creation/import script
>>
>> * Purpose
>>
>> The feature is from Yocto 1.2 Bug 1656. A script or similar system
>> that would allow you to give is an upstream URL, tarball, patch set,
>> package (SRPM, or debian style) and generate a recipe based on those
>> instructions.
>>
>> Name it with bitbake-createbb, and integrate into the build system
>> to use the build system infrastructure that similiar with
>> bitbake-runtask.
>
Hi Mark,
Thanks for your comments.
> I'd suggest changing the name to recipe-createbb or something. Since
> this isn't a "bitbake" program (or is it?)
How about name it with "create-recipe"?
>
>> * Usage:
>> bitbake-createbb<SRC_URI>
>>
>> * Design Flow
>> 1 download the source package from the SRC_URI
>> 2 get package name and version from package or tarball name.
>> 3 unpack to /tmp/bitbake-createbb-pid
>> 4 if tarball, get summary and description from freshmeat.org, README
>> files and pkgconfig files;
>> otherwise get them from SRPM spec file or debian control file
>> 5 get license, license file and checksum from LICENSE/COPYING files
>> or freshmeat.org.
>
> For license files, I would suggest we scan for common filenames:
>
> LICENSE, COPYING, COPYING.LIB, and COPYRIGHT
>
> If it doesn't match the known checksums, we tell the user and let them
> manually figure out the license type and verify things are consisten,
> but we should be able to fill out the license filename for them.
Ok, if can't get license from known checksums I'll left messages for
user to check manually.
Regards,
Kai
>
> Otherwise, it all looks fine to me.
>
> --Mark
>
>> 6 get dependent packages
>> |-- get them by parse configure files
>> |-- just parse spec file
>> 8 remove downloaded files
>>
>>
>>
>>
>>
>> bitbake-createbb
>>
>>
>> #!/usr/bin/env python
>>
>> import os
>> import sys
>> import shutil
>> import re
>> import urllib
>> import sha, md5, hashlib
>>
>> tmpdir = "/tmp/bitbake_createbb.%s" % os.getpid()
>> bin_file_name = ''
>> pkgname = ''
>> pkgversion = ''
>> summary = ''
>> description = ''
>> homepage = ''
>> license = []
>> license_files = {}
>> licenses = {}
>> depends = []
>> src_uri = ''
>> src_uri_md5sum = ''
>> src_uri_sha256sum = ''
>> inherits = []
>>
>> # support 2 package types: tar and rpm
>> pkgtype = ""
>>
>> def usage():
>> print "This is the bitbake import bb file script. Usage:\n"
>> print "\tbitbake-createbb<SRC_URI> \n"
>>
>> def download(pkg_uri):
>> try:
>> shutil.rmtree(tmpdir)
>> except:
>> pass
>>
>> cwd = os.getcwd()
>> os.mkdir(tmpdir)
>> os.chdir(tmpdir)
>> print 'Downloading package: ', pkg_uri, ' to ', tmpdir
>> ret = os.system("wget --quiet " + pkg_uri)
>> os.chdir(cwd)
>>
>> return ret
>>
>> def guess_name_by_uri(uri):
>> global bin_file_name, pkgname, pkgversion, pkgtype
>>
>> elems = uri.rsplit('/', 1)
>> pkg = bin_file_name = elems[1]
>>
>> if pkg.endswith("tgz"):
>> pkg = re.sub("tgz$", "tar.gz", pkg)
>>
>> match = re.match('(.*?)\-([0-9\.\-\~]+)\.tar', pkg)
>> if match:
>> pkgtype = 'tar'
>> pkgname = match.group(1)
>> pkgversion = match.group(2)
>>
>> match = re.match('(.*?)\-([0-9\.\~]+).*?\.src.rpm', pkg)
>> if match:
>> pkgtype = 'rpm'
>> pkgname = match.group(1)
>> pkgversion = match.group(2)
>>
>> def get_md5_sha256_sum():
>> global src_uri_md5sum, src_uri_sha256sum
>>
>> curdir = os.getcwd()
>> os.chdir(tmpdir)
>>
>> f = file(bin_file_name, 'r')
>> content = f.read()
>> f.close()
>>
>> md_five = md5.new()
>> md_five.update(content)
>> src_uri_md5sum = md_five.hexdigest()
>>
>> sha_256 = hashlib.sha256()
>> sha_256.update(content)
>> src_uri_sha256sum = sha_256.hexdigest()
>>
>> os.chdir(curdir)
>>
>> def unpack_package():
>> global bin_file_name
>>
>> cwd = os.getcwd()
>> os.chdir(tmpdir)
>> if pkgtype == 'rpm':
>> cmdline = "rpm2cpio " + bin_file_name + " | cpio -id"
>> ret = os.system(cmdline)
>> if ret != 0:
>> return ret
>> for item in os.listdir('.'):
>> if re.match(pkgname + '([0-9\.\-\~]*)\.tar', item):
>> bin_file_name = item
>> break
>>
>> cmdline = "tar axf " + bin_file_name
>> ret = os.system(cmdline)
>>
>> os.chdir(cwd)
>> return ret
>>
>> # get the fall-back description when other way fail
>> # check homepage at same time
>> def guess_description_from_readme(readme):
>> global description, homepage
>> f = file(readme)
>> state = 0
>> desc = ''
>>
>> for line in f:
>> if state == 1 and re.match('^\n', line) and len(desc)> 80:
>> state = 2
>> if state == 0 and len(line)> 1:
>> state = 1
>> if state == 1:
>> desc += line
>>
>> match = re.search('(http\:\/\/.*$name.*\.org)', line)
>> if match:
>> url = match.group(1)
>> if re.search('bug') or len(homepage)> 1:
>> pass
>> else:
>> homepage = url
>>
>> f.close()
>> if (len(desc)> 4 and len(description)< 3):
>> description = desc
>>
>> def guess_description_from_freecode(pkgname):
>> global description
>> desc = ''
>> state = 0
>>
>> html = urllib.urlopen("http://freecode.com/projects/" + pkgname)
>> for line in html:
>> if state == 1:
>> desc += line
>> if state == 0 and re.search('\<div class\=\"project-detail\"\>', line):
>> state = 1
>> if state == 1 and re.search('\<\/p\>', line):
>> state = 2
>>
>> # deal the description
>> desc = re.sub('\<p\>', '', desc)
>> desc = re.sub('\<\/p\>', '', desc)
>> desc = re.sub('\r', '', desc)
>> desc = desc.strip()
>> if len(desc)> 10:
>> description = desc
>>
>> # get Summary from pkgconfig file
>> def guess_summary_from_pc(pc):
>> global summary
>>
>> fn = file(pc)
>> for line in fn:
>> match = re.match('Description:\s*(.*)', line)
>> if match and len(summary)< 2:
>> summary = match.group(1)
>> break
>>
>> summary = summary.strip()
>> fn.close()
>>
>> def guess_description(pkgdir):
>> global pkgname, description, summary
>>
>> readmes = []
>> pcs = []
>>
>> for subdir in os.listdir(pkgdir):
>> if re.match('^README$', subdir):
>> readmes.insert(0, os.path.join(pkgdir, subdir))
>> elif re.match('README.*', subdir):
>> readmes.append(os.path.join(pkgdir, subdir))
>> elif re.match('.*\.pc.*', subdir):
>> pcs.insert(0, os.path.join(pkgdir, subdir))
>> elif re.match(pkgname + '\.pc.*', subdir):
>> pcs.insert(0, os.path.join(pkgdir, subdir))
>> elif re.match('.*\.pc', subdir):
>> pcs.append(os.path.join(pkgdir, subdir))
>>
>> for readme in readmes:
>> guess_description_from_readme(os.path.join(pkgdir, readme))
>>
>> if (len(pkgname)> 2):
>> guess_description_from_freecode(pkgname)
>>
>> for pc in pcs:
>> guess_summary_from_pc(pc)
>>
>> # if didn't get summary, use first line of description
>> if len(summary)< 2:
>> summary = description
>> summary = re.sub("\n", " ", summary)
>> summary = re.sub("\s+", " ", summary)
>> match = re.match("(.*?)\.", summary)
>> summary = match.group(1)
>>
>> # the sha1sum values are from autospectacle
>> def setup_licenses():
>> licenses['06877624ea5c77efe3b7e39b0f909eda6e25a4ec'] = "GPLv2"
>> licenses["075d599585584bb0e4b526f5c40cb6b17e0da35a"] = "GPLv2"
>> licenses["10782dd732f42f49918c839e8a5e2894c508b079"] = "GPLv2"
>> licenses["2d29c273fda30310211bbf6a24127d589be09b6c"] = "GPLv2"
>> licenses["4df5d4b947cf4e63e675729dd3f168ba844483c7"] = "LGPLv2.1"
>> licenses["503df7650052cf38efde55e85f0fe363e59b9739"] = "GPLv2"
>> licenses["5405311284eab5ab51113f87c9bfac435c695bb9"] = "GPLv2"
>> licenses["5fb362ef1680e635fe5fb212b55eef4db9ead48f"] = "LGPLv2"
>> licenses["68c94ffc34f8ad2d7bfae3f5a6b996409211c1b1"] = "GPLv2"
>> licenses["66c77efd1cf9c70d4f982ea59487b2eeb6338e26"] = "LGPLv2.1"
>> licenses["74a8a6531a42e124df07ab5599aad63870fa0bd4"] = "GPLv2"
>> licenses["8088b44375ef05202c0fca4e9e82d47591563609"] = "LGPLv2.1"
>> licenses["8624bcdae55baeef00cd11d5dfcfa60f68710a02"] = "GPLv3"
>> licenses["8e57ffebd0ed4417edc22e3f404ea3664d7fed27"] = "MIT"
>> licenses["99b5245b4714b9b89e7584bfc88da64e2d315b81"] = "BSD"
>> licenses["aba8d76d0af67d57da3c3c321caa59f3d242386b"] = "MPLv1.1"
>> licenses["bf50bac24e7ec325dbb09c6b6c4dcc88a7d79e8f"] = "LGPLv2"
>> licenses["caeb68c46fa36651acf592771d09de7937926bb3"] = "LGPLv2.1"
>> licenses["dfac199a7539a404407098a2541b9482279f690d"] = "GPLv2"
>> licenses["e60c2e780886f95df9c9ee36992b8edabec00bcc"] = "LGPLv2.1"
>> licenses["c931aad3017d975b7f20666cde0953234a9efde3"] = "GPLv2"
>>
>> def guess_licenses_from_file(copying, relname):
>> global licenses, license
>>
>> sha1 = sha.new()
>>
>> fn = open(copying)
>> content = fn.read()
>> fn.close()
>>
>> sha1.update(content)
>> digest = sha1.hexdigest()
>>
>> if digest in licenses:
>> license.append(licenses[digest])
>> md_five = md5.new()
>> md_five.update(content)
>> license_files[relname] = md_five.hexdigest()
>>
>> def guess_licenses_from_freecode():
>> global license
>>
>> lic = ''
>> state = 0
>>
>> html = urllib.urlopen("http://freecode.com/projects/" + pkgname)
>> for line in html:
>> if state == 1:
>> lic += line
>> if state == 0 and re.search('<th><span>Licenses</span></th>', line):
>> state = 1
>> if state == 1 and re.search('</a>', line):
>> state = 2
>>
>> # deal the license
>> lic = lic.strip()
>> match = re.search('<a.*?>(.*?)</a>', lic)
>> if match:
>> license.append(match.group(1))
>>
>> def guess_license(pkgdir):
>> for item in os.listdir(pkgdir):
>> realpath = os.path.join(pkgdir, item)
>> if os.path.isfile(realpath) and (re.match('COPY.*', item) \
>> or re.match('LICENSE.*', item) or re.match('GPL.*', item)):
>> guess_licenses_from_file(realpath, item)
>>
>> if len(license) == 0:
>> guess_licenses_from_freecode()
>>
>> def process_configure(pkgdir):
>> if os.path.isfile(os.path.join(pkgdir, 'autogen.sh')):
>> os.system("cd " + pkgdir + " ; ./autogen.sh&> /dev/null");
>>
>> configure = os.path.join(pkgdir, 'configure')
>> if os.path.isfile(configure):
>> if 'autotools' not in inherits:
>> inherits.append('autotools')
>>
>> fn = open(configure)
>> for line in fn:
>> match = re.match('PACKAGE_NAME=\'(.*?)\'', line)
>> if match and len(pkgname) == 0:
>> pkgname = match.group(1)
>>
>> match = re.match('PACKAGE_TARNAME=\'(.*?)\'', line)
>> if match:
>> pkgname = match.group(1)
>>
>> match = re.match('PACKAGE_VERSION=\'(.*?)\'', line)
>> if match:
>> pkgversion = match.group(1)
>>
>> match = re.match('PACKAGE_URL=\'(.*?)\'', line)
>> if match:
>> homepage = match.group(1)
>>
>> fn.close()
>>
>> def push_buildreq(dep):
>> global depends
>>
>> # remove collateral ] ) etc damage in the string
>> dep = re.sub("\"", "", dep)
>> dep = re.sub("\)", "", dep)
>> dep = re.sub("\]", "", dep)
>> dep = re.sub("\[", "", dep)
>>
>> # first, undo the space packing
>> dep = re.sub(">=", ">= ", dep)
>> dep = re.sub("<=", "<= ", dep)
>>
>> items = dep.split(' ')
>> dep = items[0]
>>
>> # don't show configure variables, we can't deal with them
>> if re.search("\$", dep):
>> return
>> if re.search("AC_SUBST", dep):
>> return
>>
>> if dep not in depends:
>> depends.append(dep)
>>
>> def parse_configure_ac(ac_file):
>> depth = 0
>> clause = ""
>>
>> fac = file(ac_file)
>>
>> for line in fac:
>> line = line.strip()
>> i = 0
>> while i< len(line):
>> if line[i] == '(':
>> depth += 1
>> if line[i] == ')' and depth> 0:
>> depth -= 1
>> clause += line[i]
>> i += 1
>> if depth> 0:
>> continue
>>
>> # remove '\n'
>> clause = re.sub('\n', '', clause)
>> clause = clause.strip()
>>
>> match = re.match('PKG_CHECK_MODULES\((.*)\)', clause)
>> if match:
>> modules = match.group(1)
>> pkg = modules.split(',', 1)[1].strip()
>> match2 = re.match('\[(.*)\]', pkg)
>> if match2:
>> pkg = match2.group(1)
>>
>> # split the build dependencies
>> pkg = pkg.replace('\s+', ' ')
>> pkg = re.sub('\s>\s', '>', pkg)
>> pkg = re.sub('\s>=\s', '>=', pkg)
>> pkg = re.sub('\s=\s', '=', pkg)
>> pkg = re.sub('\s<=\s', '<=', pkg)
>> pkg = re.sub('\s<\s', '<', pkg)
>> pkglist = pkg.split(' ')
>> for dep in pkglist:
>> push_buildreq(dep)
>>
>> match = re.match('PKG_CHECK_EXISTS\((.*)\)', clause)
>> if match:
>> exists = match.group(1)
>> pkg = exists.split(',', 1)[0].strip()
>> match2 = re.match('\[(.*)\]', pkg)
>> if match2:
>> pkg = match2.group(1)
>> pkg = pkg.strip()
>>
>> pkg = re.sub('\s+', ' ', pkg)
>> pkg = re.sub('\s>\s', '>', pkg)
>> pkg = re.sub('\s>=\s', '>=', pkg)
>> pkg = re.sub('\s=\s', '=', pkg)
>> pkg = re.sub('\s<=\s', '<=', pkg)
>> pkg = re.sub('\s<\s', '<', pkg)
>> pkglist = pkg.split(' ')
>> for dep in pkglist:
>> push_buildreq(dep)
>>
>> # these items are from autospectacle.pl
>> if re.search('_PROG_INTLTOOL', clause):
>> push_buildreq("intltool")
>> if re.search('GETTEXT_PACKAGE', clause):
>> push_buildreq('gettext')
>> if re.search('GTK_DOC_CHECK', clause):
>> push_buildreq('gtk-doc')
>> if re.search('GNOME_DOC_INIT', clause):
>> push_buildreq('gnome-doc-utils')
>> if re.search('AM_GLIB_GNU_GETTEXT', clause):
>> push_buildreq('gettext')
>>
>> match = re.search('AC_INIT\((.*)\)', clause)
>> if match:
>> ac_init = match.group(1)
>> ac_init = ac_init.strip()
>> ac_init = re.sub('\s+', ' ', ac_init)
>> version = ac_init.split(',')[1].strip()
>> if re.match('\d+(\.\d+)*', version):
>> pkgversion = version
>>
>> match = re.search('AM_INIT_AUTOMAKE\((.*)\)', clause)
>> if match:
>> am_init = match.group(1)
>> am_init = re.sub('\s+', ' ', am_init)
>> items = am_init.split(',')
>> if len(items)>= 2:
>> ver = items[1]
>> ver = re.sub('\[', '', ver)
>> ver = re.sub('\]', '', ver)
>> if re.match('\d(\.\d+)*', ver):
>> pkgversion = ver
>> clause = ''
>> fac.close
>>
>> def process_configure_ac(pkgdir):
>> configure_acs = []
>> for root, dirnames, filenames in os.walk(pkgdir):
>> for f in filenames:
>> if re.match('.*\.ac', f):
>> configure_acs.append(os.path.join(root, f))
>> if 'autotools' not in inherits:
>> inherits.append('autotools')
>>
>> for ac in configure_acs:
>> parse_configure_ac(ac)
>>
>> # check the *.pro files and get build requires from 'PKGCONFIG'
>> def process_qmake_pro(pkgdir):
>> global pkgname
>>
>> pro_files = []
>> depth = 0
>> clause = ''
>> pre_char = ''
>>
>> if os.path.isfile(os.path.join(pkgdir, pkgname + '.pro')) and
>> 'qmake2' not in inherits:
>> inherits.append('qmake2')
>> for root, dirnames, filenames in os.walk(pkgdir):
>> for f in filenames:
>> if re.match('.*\.pro', f):
>> pro_files.append(os.path.join(pkgdir, f))
>>
>> for pro in pro_files:
>> need_next_line = False
>> f = file(pro)
>> for line in f:
>> if line.startswith('#'):
>> continue
>> line = re.sub('\n', '', line)
>> clause += line
>> if line.endswith('\\'):
>> continue
>>
>> clause = clause.strip()
>> match = re.match('PKGCONFIG\s*=(.*)', clause)
>> if match:
>> dep = match.group(1)
>> push_buildreq(dep)
>> match = re.match('PKGCONFIG\s*\+=(.*)', clause)
>> if match:
>> dep = match.group(1)
>> dep = dep.strip()
>> dep = re.sub('\\\\', '', dep)
>> dep = re.sub('\s+', ' ', dep)
>> items = dep.split(' ')
>> for item in items:
>> push_buildreq(item)
>>
>> clause = ''
>>
>> f.close()
>>
>> def parse_spec_file(tmpdir):
>> # find the spec file first
>> spec_file = ''
>> for item in os.listdir(tmpdir):
>> if re.match('.*\.spec', item):
>> spec_file = os.path.join(tmpdir, item)
>> break
>>
>> if len(spec_file) == 0:
>> return 1
>>
>> global pkgname, pkgversion, summary, description, depends
>> global src_uri, homepage
>>
>> f = file(spec_file)
>> for line in f:
>> match = re.match('Summary\s*:(.*)', line)
>> if match:
>> if len(summary) == 0:
>> summary = match.group(1).strip()
>> continue
>> match = re.match('Name\s*:(.*)', line)
>> if match:
>> pkgname = match.group(1).strip()
>> continue
>> match = re.match('Version\s*:(.*)', line)
>> if match:
>> version = match.group(1).strip()
>> if re.match('\d+(\.\d+)?', version):
>> pkgversion = version
>> continue
>> match = re.match('License\s*:(.*)', line)
>> if match:
>> license.append(match.group(1).strip())
>> continue
>> match = re.match('URL(?i)\s*:(.*)', line)
>> if match:
>> homepage = match.group(1).strip()
>> continue
>> match = re.match('Source0*\s*:(.*)', line)
>> if match:
>> src_uri = match.group(1).strip()
>> src_uri = re.sub('\%\{name\}', pkgname, src_uri)
>> src_uri = re.sub('\%\{version\}', pkgversion, src_uri)
>> continue
>> match = re.match('BuildRequires\s*:(.*)', line)
>> if match:
>> deps = match.group(1).strip()
>> dpes = re.sub(',', ' ', deps)
>> deps = re.sub('\s+', ' ', deps)
>> for item in deps.split(' '):
>> push_buildreq(item)
>>
>> match = re.match('%description\s*$', line)
>> if match:
>> for l in f:
>> if l.startswith('%'):
>> break
>> description += l
>> description = re.sub('\n', '', description)
>>
>> f.close()
>>
>> def write_bbfile():
>> if len(pkgname) == 0 or len(pkgversion) == 0:
>> fail_quit("Can't get package name or version.")
>>
>> content = ''
>> content += 'DESCRIPTION = "' + description + '"\n'
>> content += 'SUMMARY = "' + summary + '"\n'
>> if len(homepage)> 0:
>> content += 'HOMEPAGE = "' + homepage + '"\n'
>>
>> content += 'LICENSE = "'
>> for lic in license:
>> content += lic + ' '
>> content += '"\n'
>> content += 'LIC_FILES_CHKSUM = "'
>> for key in license_files:
>> content += 'file://' + key + ';md5=' + license_files[key] + ' \\\n';
>> content += '\t\t"\n\n';
>>
>> if len(depends)> 0:
>> content += 'DEPENDS = "'
>> for dep in depends:
>> content += dep + ' '
>> content += ' "\n'
>>
>> content += 'PR = "r0"\n\n'
>>
>> content += 'SRC_URI = "' + src_uri + '"\n'
>> content += 'SRC_URI[md5sum] = "' + src_uri_md5sum + '"\n'
>> content += 'SRC_URI[sha256sum] = "' + src_uri_sha256sum + '"\n'
>>
>> if len(inherits)> 0:
>> content += '\ninherit '
>> for inherit in inherits:
>> content += inherit + ' '
>> content += '\n'
>>
>> bb_filename = pkgname + '_' + pkgversion + '.bb'
>> bb_file = open(bb_filename, 'w')
>> bb_file.write(content)
>> bb_file.close()
>> print 'Create bb file : ', bb_filename
>>
>> def fail_quit(msg):
>> print msg
>> clean_up()
>> exit(1)
>>
>> def clean_up():
>> shutil.rmtree(tmpdir)
>> pass
>>
>> ##########################################################
>>
>> if __name__ == '__main__':
>> if (len(sys.argv)< 2):
>> usage()
>> exit(1)
>>
>> src_uri = sys.argv[1]
>>
>> guess_name_by_uri(src_uri)
>> if pkgname is None:
>> print "Can't get the package name."
>> exit(1)
>>
>> ret = download(src_uri)
>> if ret != 0:
>> fail_quit('Download package failed. Make sure the SRC_URI is valid.')
>>
>> if unpack_package() != 0:
>> fail_quit('Unpack package failed.')
>>
>> if pkgtype == 'rpm':
>> parse_spec_file(tmpdir)
>>
>> get_md5_sha256_sum()
>>
>> pkgdir = ''
>> for subdir in os.listdir(tmpdir):
>> subdir = os.path.join(tmpdir, subdir)
>> if os.path.isdir(subdir):
>> pkgdir = subdir
>> break
>>
>> if pkgtype == 'tar':
>> parse_spec_file(pkgdir)
>>
>> guess_description(pkgdir)
>> process_configure(pkgdir)
>>
>> setup_licenses()
>> guess_license(pkgdir)
>>
>> process_configure_ac(pkgdir)
>> process_qmake_pro(pkgdir)
>>
>> write_bbfile()
>> clean_up()
>>
>>
>>
>>
>> _______________________________________________
>> poky mailing list
>> poky@yoctoproject.org
>> https://lists.yoctoproject.org/listinfo/poky
>>
>
> _______________________________________________
> poky mailing list
> poky@yoctoproject.org
> https://lists.yoctoproject.org/listinfo/poky
^ permalink raw reply [flat|nested] 5+ messages in thread
end of thread, other threads:[~2012-01-30 2:21 UTC | newest]
Thread overview: 5+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
[not found] <4F17C7D9.10506@windriver.com>
2012-01-19 9:33 ` Please help to review the Yocto 1656: Recipe creation/import script Kang Kai
2012-01-19 11:57 ` Paul Eggleton
2012-01-19 16:33 ` Mark Hatle
2012-01-30 2:21 ` Kang Kai
2012-01-20 1:57 ` Joshua Lock
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.