Please help to review the Yocto 1656: Recipe creation/import script

Please help to review the Yocto 1656: Recipe creation/import script

[Kang Kai]

[-- Attachment #1: Type: text/plain, Size: 300 bytes --]

Hi Josh,

The attachment is the the design document V2 and implement.

The V2 document changes are:
1 remove patch_set from argument list
2 deb is not support right now

Any comment is welcome.
And I will on vacation until Jan 30, so my reply may not in time.

Thanks&  Regards,
Kai



[-- Attachment #2: recipe_creation_bb.txt --]
[-- Type: text/plain, Size: 1027 bytes --]

    Design document for Recipe creation/import script

* Purpose

The feature is from Yocto 1.2 Bug 1656. A script or similar system
that would allow you to give is an upstream URL, tarball, patch set,
package (SRPM, or debian style) and generate a recipe based on those
instructions. 

Name it with bitbake-createbb, and integrate into the build system
to use the build system infrastructure that similiar with bitbake-runtask.

* Usage:
bitbake-createbb <SRC_URI> 

* Design Flow
1 download the source package from the SRC_URI
2 get package name and version from package or tarball name. 
3 unpack to /tmp/bitbake-createbb-pid
4 if tarball, get summary and description from freshmeat.org, README files and pkgconfig files;
  otherwise get them from SRPM spec file or debian control file
5 get license, license file and checksum from LICENSE/COPYING files or freshmeat.org. 
6 get dependent packages
  |-- get them by parse configure files
  |-- just parse spec file 
8 remove downloaded files




[-- Attachment #3: bitbake-createbb --]
[-- Type: text/plain, Size: 19557 bytes --]

#!/usr/bin/env python

import os
import sys
import shutil
import re
import urllib
import sha, md5, hashlib

tmpdir = "/tmp/bitbake_createbb.%s" % os.getpid()
bin_file_name = ''
pkgname = ''
pkgversion = ''
summary = ''
description = ''
homepage = ''
license = []
license_files = {}
licenses = {}
depends = []
src_uri = ''
src_uri_md5sum = ''
src_uri_sha256sum = ''
inherits = []

# support 2 package types: tar and rpm
pkgtype = ""

def usage():
    print "This is the bitbake import bb file script.  Usage:\n"
    print "\tbitbake-createbb <SRC_URI> \n"

def download(pkg_uri):
    try:
        shutil.rmtree(tmpdir)
    except:
        pass

    cwd = os.getcwd()
    os.mkdir(tmpdir)
    os.chdir(tmpdir)
    print 'Downloading package: ', pkg_uri, ' to ', tmpdir
    ret = os.system("wget --quiet " + pkg_uri)
    os.chdir(cwd)

    return ret

def guess_name_by_uri(uri):
    global bin_file_name, pkgname, pkgversion, pkgtype

    elems = uri.rsplit('/', 1)
    pkg = bin_file_name = elems[1]

    if pkg.endswith("tgz"):
        pkg = re.sub("tgz$", "tar.gz", pkg)

    match = re.match('(.*?)\-([0-9\.\-\~]+)\.tar', pkg)
    if match:
        pkgtype = 'tar'
        pkgname =  match.group(1)
        pkgversion = match.group(2)

    match = re.match('(.*?)\-([0-9\.\~]+).*?\.src.rpm', pkg)
    if match:
        pkgtype = 'rpm'
        pkgname = match.group(1)
        pkgversion = match.group(2)

def get_md5_sha256_sum():
    global src_uri_md5sum, src_uri_sha256sum

    curdir = os.getcwd()
    os.chdir(tmpdir)

    f = file(bin_file_name, 'r')
    content = f.read()
    f.close()

    md_five = md5.new()
    md_five.update(content)
    src_uri_md5sum = md_five.hexdigest()

    sha_256 = hashlib.sha256()
    sha_256.update(content)
    src_uri_sha256sum = sha_256.hexdigest()

    os.chdir(curdir)

def unpack_package():
    global bin_file_name

    cwd = os.getcwd()
    os.chdir(tmpdir)
    if pkgtype == 'rpm':
        cmdline = "rpm2cpio " + bin_file_name + " | cpio -id"
        ret = os.system(cmdline)
        if ret != 0:
            return ret
        for item in os.listdir('.'):
            if re.match(pkgname + '([0-9\.\-\~]*)\.tar', item):
                bin_file_name = item
                break

    cmdline = "tar axf " + bin_file_name
    ret = os.system(cmdline)

    os.chdir(cwd)
    return ret

# get the fall-back description when other way fail
# check homepage at same time
def guess_description_from_readme(readme):
    global description, homepage
    f = file(readme)
    state = 0
    desc = ''

    for line in f:
        if state == 1 and re.match('^\n', line) and len(desc) > 80:
            state = 2
        if state == 0 and len(line) > 1:
            state = 1
        if state == 1:
            desc += line

        match = re.search('(http\:\/\/.*$name.*\.org)', line)
        if match:
            url = match.group(1)
            if re.search('bug') or len(homepage) > 1:
                pass
            else:
                homepage = url

    f.close()
    if (len(desc) > 4 and len(description) < 3):
        description = desc

def guess_description_from_freecode(pkgname):
    global description
    desc = ''
    state = 0

    html = urllib.urlopen("http://freecode.com/projects/" + pkgname)
    for line in html:
        if state == 1:
            desc += line
        if state == 0 and re.search('\<div class\=\"project-detail\"\>', line):
            state = 1
        if state == 1 and re.search('\<\/p\>', line):
            state = 2

    # deal the description
    desc = re.sub('\<p\>', '', desc)
    desc = re.sub('\<\/p\>', '', desc)
    desc = re.sub('\r', '', desc)
    desc = desc.strip()
    if len(desc) > 10:
       description = desc

# get Summary from pkgconfig file
def guess_summary_from_pc(pc):
    global summary

    fn = file(pc)
    for line in fn:
        match = re.match('Description:\s*(.*)', line)
        if match and len(summary) < 2:
            summary = match.group(1)
            break

    summary = summary.strip()
    fn.close()

def guess_description(pkgdir):
    global pkgname, description, summary

    readmes = []
    pcs = []

    for subdir in os.listdir(pkgdir):
        if re.match('^README$', subdir):
            readmes.insert(0, os.path.join(pkgdir, subdir))
        elif re.match('README.*', subdir):
            readmes.append(os.path.join(pkgdir, subdir))
        elif re.match('.*\.pc.*', subdir):
            pcs.insert(0, os.path.join(pkgdir, subdir))
        elif re.match(pkgname + '\.pc.*', subdir):
            pcs.insert(0, os.path.join(pkgdir, subdir))
        elif re.match('.*\.pc', subdir):
            pcs.append(os.path.join(pkgdir, subdir))

    for readme in readmes:
        guess_description_from_readme(os.path.join(pkgdir, readme))

    if (len(pkgname) > 2):
        guess_description_from_freecode(pkgname)

    for pc in pcs:
        guess_summary_from_pc(pc)

    # if didn't get summary, use first line of description
    if len(summary) < 2:
        summary = description
        summary = re.sub("\n", " ", summary)
        summary = re.sub("\s+", " ", summary)
        match = re.match("(.*?)\.", summary)
        summary = match.group(1)

# the sha1sum values are from autospectacle
def setup_licenses():
    licenses['06877624ea5c77efe3b7e39b0f909eda6e25a4ec'] =  "GPLv2"
    licenses["075d599585584bb0e4b526f5c40cb6b17e0da35a"] = "GPLv2"
    licenses["10782dd732f42f49918c839e8a5e2894c508b079"] = "GPLv2"
    licenses["2d29c273fda30310211bbf6a24127d589be09b6c"] = "GPLv2"
    licenses["4df5d4b947cf4e63e675729dd3f168ba844483c7"] = "LGPLv2.1"
    licenses["503df7650052cf38efde55e85f0fe363e59b9739"] = "GPLv2"
    licenses["5405311284eab5ab51113f87c9bfac435c695bb9"] = "GPLv2"
    licenses["5fb362ef1680e635fe5fb212b55eef4db9ead48f"] = "LGPLv2"
    licenses["68c94ffc34f8ad2d7bfae3f5a6b996409211c1b1"] = "GPLv2"
    licenses["66c77efd1cf9c70d4f982ea59487b2eeb6338e26"] = "LGPLv2.1"
    licenses["74a8a6531a42e124df07ab5599aad63870fa0bd4"] = "GPLv2"
    licenses["8088b44375ef05202c0fca4e9e82d47591563609"] = "LGPLv2.1"
    licenses["8624bcdae55baeef00cd11d5dfcfa60f68710a02"] = "GPLv3"
    licenses["8e57ffebd0ed4417edc22e3f404ea3664d7fed27"] = "MIT"
    licenses["99b5245b4714b9b89e7584bfc88da64e2d315b81"] = "BSD"
    licenses["aba8d76d0af67d57da3c3c321caa59f3d242386b"] = "MPLv1.1"
    licenses["bf50bac24e7ec325dbb09c6b6c4dcc88a7d79e8f"] = "LGPLv2"
    licenses["caeb68c46fa36651acf592771d09de7937926bb3"] = "LGPLv2.1"
    licenses["dfac199a7539a404407098a2541b9482279f690d"] = "GPLv2"
    licenses["e60c2e780886f95df9c9ee36992b8edabec00bcc"] = "LGPLv2.1"
    licenses["c931aad3017d975b7f20666cde0953234a9efde3"] = "GPLv2"

def guess_licenses_from_file(copying, relname):
    global licenses, license

    sha1 = sha.new()

    fn = open(copying)
    content = fn.read()
    fn.close()

    sha1.update(content)
    digest = sha1.hexdigest()

    if digest in licenses:
        license.append(licenses[digest])
        md_five = md5.new()
        md_five.update(content)
        license_files[relname] = md_five.hexdigest()

def guess_licenses_from_freecode():
    global license

    lic = ''
    state = 0

    html = urllib.urlopen("http://freecode.com/projects/" + pkgname)
    for line in html:
        if state == 1:
            lic += line
        if state == 0 and re.search('<th><span>Licenses</span></th>', line):
            state = 1
        if state == 1 and re.search('</a>', line):
            state = 2

    # deal the license
    lic = lic.strip()
    match = re.search('<a.*?>(.*?)</a>', lic)
    if match:
        license.append(match.group(1))

def guess_license(pkgdir):
    for item in  os.listdir(pkgdir):
        realpath = os.path.join(pkgdir, item)
        if os.path.isfile(realpath) and (re.match('COPY.*', item) \
                or re.match('LICENSE.*', item) or re.match('GPL.*', item)):
            guess_licenses_from_file(realpath, item)

    if len(license) == 0:
        guess_licenses_from_freecode()

def process_configure(pkgdir):
    if os.path.isfile(os.path.join(pkgdir, 'autogen.sh')):
        os.system("cd " + pkgdir + " ; ./autogen.sh &> /dev/null");

    configure = os.path.join(pkgdir, 'configure')
    if os.path.isfile(configure):
        if 'autotools' not in inherits:
            inherits.append('autotools')

        fn = open(configure)
        for line in fn:
            match = re.match('PACKAGE_NAME=\'(.*?)\'', line)
            if match and len(pkgname) == 0:
                pkgname = match.group(1)

            match = re.match('PACKAGE_TARNAME=\'(.*?)\'', line)
            if match:
                pkgname = match.group(1)

            match = re.match('PACKAGE_VERSION=\'(.*?)\'', line)
            if match:
                pkgversion = match.group(1)

            match = re.match('PACKAGE_URL=\'(.*?)\'', line)
            if match:
                homepage = match.group(1)

        fn.close()

def push_buildreq(dep):
    global depends

    # remove collateral ] ) etc damage in the string
    dep = re.sub("\"", "", dep)
    dep = re.sub("\)", "", dep)
    dep = re.sub("\]", "", dep)
    dep = re.sub("\[", "", dep)

    # first, undo the space packing
    dep = re.sub(">=", " >= ", dep)
    dep = re.sub("<=", " <= ", dep)

    items = dep.split(' ')
    dep = items[0]

    # don't show configure variables, we can't deal with them
    if re.search("\$", dep):
        return
    if re.search("AC_SUBST", dep):
        return

    if dep not in depends:
        depends.append(dep)

def parse_configure_ac(ac_file):
    depth = 0
    clause = ""

    fac = file(ac_file)

    for line in fac:
        line = line.strip()
        i = 0
        while i < len(line):
            if line[i] == '(':
                depth += 1
            if line[i] == ')' and depth > 0:
                depth -= 1
            clause += line[i]
            i += 1
        if depth > 0:
            continue

        # remove '\n'
        clause = re.sub('\n', '', clause)
        clause = clause.strip()

        match = re.match('PKG_CHECK_MODULES\((.*)\)', clause)
        if match:
            modules = match.group(1)
            pkg = modules.split(',', 1)[1].strip()
            match2 = re.match('\[(.*)\]', pkg)
            if match2:
                pkg = match2.group(1)

            # split the build dependencies
            pkg = pkg.replace('\s+', ' ')
            pkg = re.sub('\s>\s', '>', pkg)
            pkg = re.sub('\s>=\s', '>=', pkg)
            pkg = re.sub('\s=\s', '=', pkg)
            pkg = re.sub('\s<=\s', '<=', pkg)
            pkg = re.sub('\s<\s', '<', pkg)
            pkglist = pkg.split(' ')
            for dep in pkglist:
                push_buildreq(dep)

        match = re.match('PKG_CHECK_EXISTS\((.*)\)', clause)
        if match:
            exists = match.group(1)
            pkg = exists.split(',', 1)[0].strip()
            match2 = re.match('\[(.*)\]', pkg)
            if match2:
                pkg = match2.group(1)
                pkg = pkg.strip()

            pkg = re.sub('\s+', ' ', pkg)
            pkg = re.sub('\s>\s', '>', pkg)
            pkg = re.sub('\s>=\s', '>=', pkg)
            pkg = re.sub('\s=\s', '=', pkg)
            pkg = re.sub('\s<=\s', '<=', pkg)
            pkg = re.sub('\s<\s', '<', pkg)
            pkglist = pkg.split(' ')
            for dep in pkglist:
                push_buildreq(dep)

        # these items are from autospectacle.pl
        if re.search('_PROG_INTLTOOL', clause):
            push_buildreq("intltool")
        if re.search('GETTEXT_PACKAGE', clause):
            push_buildreq('gettext')
        if re.search('GTK_DOC_CHECK', clause):
            push_buildreq('gtk-doc')
        if re.search('GNOME_DOC_INIT', clause):
            push_buildreq('gnome-doc-utils')
        if re.search('AM_GLIB_GNU_GETTEXT', clause):
            push_buildreq('gettext')

        match = re.search('AC_INIT\((.*)\)', clause)
        if match:
            ac_init = match.group(1)
            ac_init = ac_init.strip()
            ac_init = re.sub('\s+', ' ', ac_init)
            version = ac_init.split(',')[1].strip()
            if re.match('\d+(\.\d+)*', version):
                pkgversion = version

        match = re.search('AM_INIT_AUTOMAKE\((.*)\)', clause)
        if match:
            am_init = match.group(1)
            am_init = re.sub('\s+', ' ', am_init)
            items = am_init.split(',')
            if len(items) >= 2:
                ver = items[1]
                ver = re.sub('\[', '', ver)
                ver = re.sub('\]', '', ver)
                if re.match('\d(\.\d+)*', ver):
                    pkgversion = ver
        clause = ''
    fac.close

def process_configure_ac(pkgdir):
    configure_acs = []
    for root, dirnames, filenames in os.walk(pkgdir):
        for f in filenames:
            if re.match('.*\.ac', f):
                configure_acs.append(os.path.join(root, f))
                if 'autotools' not in inherits:
                    inherits.append('autotools')

    for ac in configure_acs:
        parse_configure_ac(ac)

# check the *.pro files and get build requires from 'PKGCONFIG'
def process_qmake_pro(pkgdir):
    global pkgname

    pro_files = []
    depth = 0
    clause = ''
    pre_char = ''

    if os.path.isfile(os.path.join(pkgdir, pkgname + '.pro')) and 'qmake2' not in inherits:
        inherits.append('qmake2')
    for root, dirnames, filenames in os.walk(pkgdir):
        for f in filenames:
            if re.match('.*\.pro', f):
                pro_files.append(os.path.join(pkgdir, f))

    for pro in pro_files:
        need_next_line = False
        f = file(pro)
        for line in f:
            if line.startswith('#'):
                continue
            line = re.sub('\n', '', line)
            clause += line
            if line.endswith('\\'):
                continue

            clause = clause.strip()
            match =  re.match('PKGCONFIG\s*=(.*)', clause)
            if match:
                dep = match.group(1)
                push_buildreq(dep)
            match = re.match('PKGCONFIG\s*\+=(.*)', clause)
            if match:
                dep = match.group(1)
                dep = dep.strip()
                dep = re.sub('\\\\', '', dep)
                dep = re.sub('\s+', ' ', dep)
                items = dep.split(' ')
                for item in items:
                    push_buildreq(item)

            clause = ''

        f.close()

def parse_spec_file(tmpdir):
    # find the spec file first
    spec_file = ''
    for item in os.listdir(tmpdir):
        if re.match('.*\.spec', item):
            spec_file = os.path.join(tmpdir, item)
            break

    if len(spec_file) == 0:
        return 1

    global pkgname, pkgversion, summary, description, depends
    global src_uri, homepage

    f = file(spec_file)
    for line in f:
        match = re.match('Summary\s*:(.*)', line)
        if match:
            if len(summary) == 0:
                summary = match.group(1).strip()
            continue
        match = re.match('Name\s*:(.*)', line)
        if match:
            pkgname = match.group(1).strip()
            continue
        match = re.match('Version\s*:(.*)', line)
        if match:
            version = match.group(1).strip()
            if re.match('\d+(\.\d+)?', version):
                pkgversion = version
            continue
        match = re.match('License\s*:(.*)', line)
        if match:
            license.append(match.group(1).strip())
            continue
        match = re.match('URL(?i)\s*:(.*)', line)
        if match:
            homepage = match.group(1).strip()
            continue
        match = re.match('Source0*\s*:(.*)', line)
        if match:
            src_uri = match.group(1).strip()
            src_uri = re.sub('\%\{name\}', pkgname, src_uri)
            src_uri = re.sub('\%\{version\}', pkgversion, src_uri)
            continue
        match = re.match('BuildRequires\s*:(.*)', line)
        if match:
            deps = match.group(1).strip()
            dpes = re.sub(',', ' ', deps)
            deps = re.sub('\s+', ' ', deps)
            for item in deps.split(' '):
                push_buildreq(item)

        match = re.match('%description\s*$', line)
        if match:
            for l in f:
                if l.startswith('%'):
                    break
                description += l
            description = re.sub('\n', '', description)

    f.close()

def write_bbfile():
    if len(pkgname) == 0 or len(pkgversion) == 0:
        fail_quit("Can't get package name or version.")

    content = ''
    content += 'DESCRIPTION = "' + description + '"\n'
    content += 'SUMMARY = "' + summary + '"\n'
    if len(homepage) > 0:
        content += 'HOMEPAGE = "' + homepage + '"\n'

    content += 'LICENSE = "'
    for lic in license:
        content += lic + ' '
    content += '"\n'
    content += 'LIC_FILES_CHKSUM = "'
    for key in license_files:
        content += 'file://' + key + ';md5=' + license_files[key] + ' \\\n';
    content += '\t\t"\n\n';

    if len(depends) > 0:
        content += 'DEPENDS = "'
        for dep in depends:
            content += dep + ' '
        content += ' "\n'

    content += 'PR = "r0"\n\n'

    content += 'SRC_URI = "' + src_uri + '"\n'
    content += 'SRC_URI[md5sum] = "' + src_uri_md5sum + '"\n'
    content += 'SRC_URI[sha256sum] = "' + src_uri_sha256sum + '"\n'

    if len(inherits) > 0:
        content += '\ninherit '
        for inherit in inherits:
            content += inherit + ' '
        content += '\n'

    bb_filename = pkgname + '_' + pkgversion + '.bb'
    bb_file = open(bb_filename, 'w')
    bb_file.write(content)
    bb_file.close()
    print 'Create bb file : ', bb_filename

def fail_quit(msg):
    print msg
    clean_up()
    exit(1)

def clean_up():
    shutil.rmtree(tmpdir)
    pass

##########################################################

if __name__ == '__main__':
    if (len(sys.argv) < 2):
        usage()
        exit(1)

    src_uri = sys.argv[1]

    guess_name_by_uri(src_uri)
    if pkgname is None:
        print "Can't get the package name."
        exit(1)

    ret = download(src_uri)
    if ret != 0:
        fail_quit('Download package failed. Make sure the SRC_URI is valid.')

    if unpack_package() != 0:
        fail_quit('Unpack package failed.')

    if pkgtype == 'rpm':
        parse_spec_file(tmpdir)

    get_md5_sha256_sum()

    pkgdir = ''
    for subdir in os.listdir(tmpdir):
        subdir = os.path.join(tmpdir, subdir)
        if os.path.isdir(subdir):
            pkgdir = subdir
            break

    if pkgtype == 'tar':
        parse_spec_file(pkgdir)

    guess_description(pkgdir)
    process_configure(pkgdir)

    setup_licenses()
    guess_license(pkgdir)

    process_configure_ac(pkgdir)
    process_qmake_pro(pkgdir)

    write_bbfile()
    clean_up()

Re: Please help to review the Yocto 1656: Recipe creation/import script

[Paul Eggleton]

Hi Kai,

On Thursday 19 January 2012 17:33:47 Kang Kai wrote:
> The attachment is the the design document V2 and implement.
> 
> The V2 document changes are:
> 1 remove patch_set from argument list
> 2 deb is not support right now
> 
> Any comment is welcome.
> And I will on vacation until Jan 30, so my reply may not in time.

Great work! I can see this being really useful to help people get started with 
new recipes. A few comments:

1) I tried it on hdparm and it worked very well. However, obexftp caused an 
error:

----------------
Downloading package:  
http://downloads.sourceforge.net/openobex/obexftp-0.23.tar.bz2  to  
/tmp/bitbake_createbb.5729
Traceback (most recent call last):
  File "./bitbake-createbb", line 643, in <module>
    process_configure(pkgdir)
  File "./bitbake-createbb", line 295, in process_configure
    if match and len(pkgname) == 0:
UnboundLocalError: local variable 'pkgname' referenced before assignment
----------------

2) If it is able to find license file(s) but unable to find any appropriate for 
LIC_FILES_CHKSUM, it would be nice if it would print a warning just so the 
user knows they will need to set this themselves (bonus points if you still 
list LIC_FILES_CHKSUM entries for license files that did not match the known 
checksums either in a comment in the output or in the warning message, to save 
the user work if they know the files are appropriate). If you prefer I can send 
a patch afterwards for this.

3) If you have an RPM spec file it is possible to extract PACKAGES and FILES 
and possibly even EXTRA_OECONF. Somewhere around here I have a very basic 
script to do this I think, I could send it to you if you like (or we can leave 
this for later).

Cheers,
Paul

-- 

Paul Eggleton
Intel Open Source Technology Centre

Re: Please help to review the Yocto 1656: Recipe creation/import script

[Mark Hatle]

On 1/19/12 3:33 AM, Kang Kai wrote:
> Hi Josh,
>
> The attachment is the the design document V2 and implement.
>
> The V2 document changes are:
> 1 remove patch_set from argument list
> 2 deb is not support right now
>
> Any comment is welcome.
> And I will on vacation until Jan 30, so my reply may not in time.
>
> Thanks&   Regards,
> Kai
>
>
>
>
> recipe_creation_bb.txt
>
>
>      Design document for Recipe creation/import script
>
> * Purpose
>
> The feature is from Yocto 1.2 Bug 1656. A script or similar system
> that would allow you to give is an upstream URL, tarball, patch set,
> package (SRPM, or debian style) and generate a recipe based on those
> instructions.
>
> Name it with bitbake-createbb, and integrate into the build system
> to use the build system infrastructure that similiar with bitbake-runtask.

I'd suggest changing the name to recipe-createbb or something.  Since this isn't 
a "bitbake" program (or is it?)

> * Usage:
> bitbake-createbb<SRC_URI>
>
> * Design Flow
> 1 download the source package from the SRC_URI
> 2 get package name and version from package or tarball name.
> 3 unpack to /tmp/bitbake-createbb-pid
> 4 if tarball, get summary and description from freshmeat.org, README files and pkgconfig files;
>    otherwise get them from SRPM spec file or debian control file
> 5 get license, license file and checksum from LICENSE/COPYING files or freshmeat.org.

For license files, I would suggest we scan for common filenames:

LICENSE, COPYING, COPYING.LIB, and COPYRIGHT

If it doesn't match the known checksums, we tell the user and let them manually 
figure out the license type and verify things are consisten, but we should be 
able to fill out the license filename for them.

Otherwise, it all looks fine to me.

--Mark

> 6 get dependent packages
>    |-- get them by parse configure files
>    |-- just parse spec file
> 8 remove downloaded files
>
>
>
>
>
> bitbake-createbb
>
>
> #!/usr/bin/env python
>
> import os
> import sys
> import shutil
> import re
> import urllib
> import sha, md5, hashlib
>
> tmpdir = "/tmp/bitbake_createbb.%s" % os.getpid()
> bin_file_name = ''
> pkgname = ''
> pkgversion = ''
> summary = ''
> description = ''
> homepage = ''
> license = []
> license_files = {}
> licenses = {}
> depends = []
> src_uri = ''
> src_uri_md5sum = ''
> src_uri_sha256sum = ''
> inherits = []
>
> # support 2 package types: tar and rpm
> pkgtype = ""
>
> def usage():
>      print "This is the bitbake import bb file script.  Usage:\n"
>      print "\tbitbake-createbb<SRC_URI>  \n"
>
> def download(pkg_uri):
>      try:
>          shutil.rmtree(tmpdir)
>      except:
>          pass
>
>      cwd = os.getcwd()
>      os.mkdir(tmpdir)
>      os.chdir(tmpdir)
>      print 'Downloading package: ', pkg_uri, ' to ', tmpdir
>      ret = os.system("wget --quiet " + pkg_uri)
>      os.chdir(cwd)
>
>      return ret
>
> def guess_name_by_uri(uri):
>      global bin_file_name, pkgname, pkgversion, pkgtype
>
>      elems = uri.rsplit('/', 1)
>      pkg = bin_file_name = elems[1]
>
>      if pkg.endswith("tgz"):
>          pkg = re.sub("tgz$", "tar.gz", pkg)
>
>      match = re.match('(.*?)\-([0-9\.\-\~]+)\.tar', pkg)
>      if match:
>          pkgtype = 'tar'
>          pkgname =  match.group(1)
>          pkgversion = match.group(2)
>
>      match = re.match('(.*?)\-([0-9\.\~]+).*?\.src.rpm', pkg)
>      if match:
>          pkgtype = 'rpm'
>          pkgname = match.group(1)
>          pkgversion = match.group(2)
>
> def get_md5_sha256_sum():
>      global src_uri_md5sum, src_uri_sha256sum
>
>      curdir = os.getcwd()
>      os.chdir(tmpdir)
>
>      f = file(bin_file_name, 'r')
>      content = f.read()
>      f.close()
>
>      md_five = md5.new()
>      md_five.update(content)
>      src_uri_md5sum = md_five.hexdigest()
>
>      sha_256 = hashlib.sha256()
>      sha_256.update(content)
>      src_uri_sha256sum = sha_256.hexdigest()
>
>      os.chdir(curdir)
>
> def unpack_package():
>      global bin_file_name
>
>      cwd = os.getcwd()
>      os.chdir(tmpdir)
>      if pkgtype == 'rpm':
>          cmdline = "rpm2cpio " + bin_file_name + " | cpio -id"
>          ret = os.system(cmdline)
>          if ret != 0:
>              return ret
>          for item in os.listdir('.'):
>              if re.match(pkgname + '([0-9\.\-\~]*)\.tar', item):
>                  bin_file_name = item
>                  break
>
>      cmdline = "tar axf " + bin_file_name
>      ret = os.system(cmdline)
>
>      os.chdir(cwd)
>      return ret
>
> # get the fall-back description when other way fail
> # check homepage at same time
> def guess_description_from_readme(readme):
>      global description, homepage
>      f = file(readme)
>      state = 0
>      desc = ''
>
>      for line in f:
>          if state == 1 and re.match('^\n', line) and len(desc)>  80:
>              state = 2
>          if state == 0 and len(line)>  1:
>              state = 1
>          if state == 1:
>              desc += line
>
>          match = re.search('(http\:\/\/.*$name.*\.org)', line)
>          if match:
>              url = match.group(1)
>              if re.search('bug') or len(homepage)>  1:
>                  pass
>              else:
>                  homepage = url
>
>      f.close()
>      if (len(desc)>  4 and len(description)<  3):
>          description = desc
>
> def guess_description_from_freecode(pkgname):
>      global description
>      desc = ''
>      state = 0
>
>      html = urllib.urlopen("http://freecode.com/projects/"  + pkgname)
>      for line in html:
>          if state == 1:
>              desc += line
>          if state == 0 and re.search('\<div class\=\"project-detail\"\>', line):
>              state = 1
>          if state == 1 and re.search('\<\/p\>', line):
>              state = 2
>
>      # deal the description
>      desc = re.sub('\<p\>', '', desc)
>      desc = re.sub('\<\/p\>', '', desc)
>      desc = re.sub('\r', '', desc)
>      desc = desc.strip()
>      if len(desc)>  10:
>         description = desc
>
> # get Summary from pkgconfig file
> def guess_summary_from_pc(pc):
>      global summary
>
>      fn = file(pc)
>      for line in fn:
>          match = re.match('Description:\s*(.*)', line)
>          if match and len(summary)<  2:
>              summary = match.group(1)
>              break
>
>      summary = summary.strip()
>      fn.close()
>
> def guess_description(pkgdir):
>      global pkgname, description, summary
>
>      readmes = []
>      pcs = []
>
>      for subdir in os.listdir(pkgdir):
>          if re.match('^README$', subdir):
>              readmes.insert(0, os.path.join(pkgdir, subdir))
>          elif re.match('README.*', subdir):
>              readmes.append(os.path.join(pkgdir, subdir))
>          elif re.match('.*\.pc.*', subdir):
>              pcs.insert(0, os.path.join(pkgdir, subdir))
>          elif re.match(pkgname + '\.pc.*', subdir):
>              pcs.insert(0, os.path.join(pkgdir, subdir))
>          elif re.match('.*\.pc', subdir):
>              pcs.append(os.path.join(pkgdir, subdir))
>
>      for readme in readmes:
>          guess_description_from_readme(os.path.join(pkgdir, readme))
>
>      if (len(pkgname)>  2):
>          guess_description_from_freecode(pkgname)
>
>      for pc in pcs:
>          guess_summary_from_pc(pc)
>
>      # if didn't get summary, use first line of description
>      if len(summary)<  2:
>          summary = description
>          summary = re.sub("\n", " ", summary)
>          summary = re.sub("\s+", " ", summary)
>          match = re.match("(.*?)\.", summary)
>          summary = match.group(1)
>
> # the sha1sum values are from autospectacle
> def setup_licenses():
>      licenses['06877624ea5c77efe3b7e39b0f909eda6e25a4ec'] =  "GPLv2"
>      licenses["075d599585584bb0e4b526f5c40cb6b17e0da35a"] = "GPLv2"
>      licenses["10782dd732f42f49918c839e8a5e2894c508b079"] = "GPLv2"
>      licenses["2d29c273fda30310211bbf6a24127d589be09b6c"] = "GPLv2"
>      licenses["4df5d4b947cf4e63e675729dd3f168ba844483c7"] = "LGPLv2.1"
>      licenses["503df7650052cf38efde55e85f0fe363e59b9739"] = "GPLv2"
>      licenses["5405311284eab5ab51113f87c9bfac435c695bb9"] = "GPLv2"
>      licenses["5fb362ef1680e635fe5fb212b55eef4db9ead48f"] = "LGPLv2"
>      licenses["68c94ffc34f8ad2d7bfae3f5a6b996409211c1b1"] = "GPLv2"
>      licenses["66c77efd1cf9c70d4f982ea59487b2eeb6338e26"] = "LGPLv2.1"
>      licenses["74a8a6531a42e124df07ab5599aad63870fa0bd4"] = "GPLv2"
>      licenses["8088b44375ef05202c0fca4e9e82d47591563609"] = "LGPLv2.1"
>      licenses["8624bcdae55baeef00cd11d5dfcfa60f68710a02"] = "GPLv3"
>      licenses["8e57ffebd0ed4417edc22e3f404ea3664d7fed27"] = "MIT"
>      licenses["99b5245b4714b9b89e7584bfc88da64e2d315b81"] = "BSD"
>      licenses["aba8d76d0af67d57da3c3c321caa59f3d242386b"] = "MPLv1.1"
>      licenses["bf50bac24e7ec325dbb09c6b6c4dcc88a7d79e8f"] = "LGPLv2"
>      licenses["caeb68c46fa36651acf592771d09de7937926bb3"] = "LGPLv2.1"
>      licenses["dfac199a7539a404407098a2541b9482279f690d"] = "GPLv2"
>      licenses["e60c2e780886f95df9c9ee36992b8edabec00bcc"] = "LGPLv2.1"
>      licenses["c931aad3017d975b7f20666cde0953234a9efde3"] = "GPLv2"
>
> def guess_licenses_from_file(copying, relname):
>      global licenses, license
>
>      sha1 = sha.new()
>
>      fn = open(copying)
>      content = fn.read()
>      fn.close()
>
>      sha1.update(content)
>      digest = sha1.hexdigest()
>
>      if digest in licenses:
>          license.append(licenses[digest])
>          md_five = md5.new()
>          md_five.update(content)
>          license_files[relname] = md_five.hexdigest()
>
> def guess_licenses_from_freecode():
>      global license
>
>      lic = ''
>      state = 0
>
>      html = urllib.urlopen("http://freecode.com/projects/"  + pkgname)
>      for line in html:
>          if state == 1:
>              lic += line
>          if state == 0 and re.search('<th><span>Licenses</span></th>', line):
>              state = 1
>          if state == 1 and re.search('</a>', line):
>              state = 2
>
>      # deal the license
>      lic = lic.strip()
>      match = re.search('<a.*?>(.*?)</a>', lic)
>      if match:
>          license.append(match.group(1))
>
> def guess_license(pkgdir):
>      for item in  os.listdir(pkgdir):
>          realpath = os.path.join(pkgdir, item)
>          if os.path.isfile(realpath) and (re.match('COPY.*', item) \
>                  or re.match('LICENSE.*', item) or re.match('GPL.*', item)):
>              guess_licenses_from_file(realpath, item)
>
>      if len(license) == 0:
>          guess_licenses_from_freecode()
>
> def process_configure(pkgdir):
>      if os.path.isfile(os.path.join(pkgdir, 'autogen.sh')):
>          os.system("cd " + pkgdir + " ; ./autogen.sh&>  /dev/null");
>
>      configure = os.path.join(pkgdir, 'configure')
>      if os.path.isfile(configure):
>          if 'autotools' not in inherits:
>              inherits.append('autotools')
>
>          fn = open(configure)
>          for line in fn:
>              match = re.match('PACKAGE_NAME=\'(.*?)\'', line)
>              if match and len(pkgname) == 0:
>                  pkgname = match.group(1)
>
>              match = re.match('PACKAGE_TARNAME=\'(.*?)\'', line)
>              if match:
>                  pkgname = match.group(1)
>
>              match = re.match('PACKAGE_VERSION=\'(.*?)\'', line)
>              if match:
>                  pkgversion = match.group(1)
>
>              match = re.match('PACKAGE_URL=\'(.*?)\'', line)
>              if match:
>                  homepage = match.group(1)
>
>          fn.close()
>
> def push_buildreq(dep):
>      global depends
>
>      # remove collateral ] ) etc damage in the string
>      dep = re.sub("\"", "", dep)
>      dep = re.sub("\)", "", dep)
>      dep = re.sub("\]", "", dep)
>      dep = re.sub("\[", "", dep)
>
>      # first, undo the space packing
>      dep = re.sub(">=", ">= ", dep)
>      dep = re.sub("<=", "<= ", dep)
>
>      items = dep.split(' ')
>      dep = items[0]
>
>      # don't show configure variables, we can't deal with them
>      if re.search("\$", dep):
>          return
>      if re.search("AC_SUBST", dep):
>          return
>
>      if dep not in depends:
>          depends.append(dep)
>
> def parse_configure_ac(ac_file):
>      depth = 0
>      clause = ""
>
>      fac = file(ac_file)
>
>      for line in fac:
>          line = line.strip()
>          i = 0
>          while i<  len(line):
>              if line[i] == '(':
>                  depth += 1
>              if line[i] == ')' and depth>  0:
>                  depth -= 1
>              clause += line[i]
>              i += 1
>          if depth>  0:
>              continue
>
>          # remove '\n'
>          clause = re.sub('\n', '', clause)
>          clause = clause.strip()
>
>          match = re.match('PKG_CHECK_MODULES\((.*)\)', clause)
>          if match:
>              modules = match.group(1)
>              pkg = modules.split(',', 1)[1].strip()
>              match2 = re.match('\[(.*)\]', pkg)
>              if match2:
>                  pkg = match2.group(1)
>
>              # split the build dependencies
>              pkg = pkg.replace('\s+', ' ')
>              pkg = re.sub('\s>\s', '>', pkg)
>              pkg = re.sub('\s>=\s', '>=', pkg)
>              pkg = re.sub('\s=\s', '=', pkg)
>              pkg = re.sub('\s<=\s', '<=', pkg)
>              pkg = re.sub('\s<\s', '<', pkg)
>              pkglist = pkg.split(' ')
>              for dep in pkglist:
>                  push_buildreq(dep)
>
>          match = re.match('PKG_CHECK_EXISTS\((.*)\)', clause)
>          if match:
>              exists = match.group(1)
>              pkg = exists.split(',', 1)[0].strip()
>              match2 = re.match('\[(.*)\]', pkg)
>              if match2:
>                  pkg = match2.group(1)
>                  pkg = pkg.strip()
>
>              pkg = re.sub('\s+', ' ', pkg)
>              pkg = re.sub('\s>\s', '>', pkg)
>              pkg = re.sub('\s>=\s', '>=', pkg)
>              pkg = re.sub('\s=\s', '=', pkg)
>              pkg = re.sub('\s<=\s', '<=', pkg)
>              pkg = re.sub('\s<\s', '<', pkg)
>              pkglist = pkg.split(' ')
>              for dep in pkglist:
>                  push_buildreq(dep)
>
>          # these items are from autospectacle.pl
>          if re.search('_PROG_INTLTOOL', clause):
>              push_buildreq("intltool")
>          if re.search('GETTEXT_PACKAGE', clause):
>              push_buildreq('gettext')
>          if re.search('GTK_DOC_CHECK', clause):
>              push_buildreq('gtk-doc')
>          if re.search('GNOME_DOC_INIT', clause):
>              push_buildreq('gnome-doc-utils')
>          if re.search('AM_GLIB_GNU_GETTEXT', clause):
>              push_buildreq('gettext')
>
>          match = re.search('AC_INIT\((.*)\)', clause)
>          if match:
>              ac_init = match.group(1)
>              ac_init = ac_init.strip()
>              ac_init = re.sub('\s+', ' ', ac_init)
>              version = ac_init.split(',')[1].strip()
>              if re.match('\d+(\.\d+)*', version):
>                  pkgversion = version
>
>          match = re.search('AM_INIT_AUTOMAKE\((.*)\)', clause)
>          if match:
>              am_init = match.group(1)
>              am_init = re.sub('\s+', ' ', am_init)
>              items = am_init.split(',')
>              if len(items)>= 2:
>                  ver = items[1]
>                  ver = re.sub('\[', '', ver)
>                  ver = re.sub('\]', '', ver)
>                  if re.match('\d(\.\d+)*', ver):
>                      pkgversion = ver
>          clause = ''
>      fac.close
>
> def process_configure_ac(pkgdir):
>      configure_acs = []
>      for root, dirnames, filenames in os.walk(pkgdir):
>          for f in filenames:
>              if re.match('.*\.ac', f):
>                  configure_acs.append(os.path.join(root, f))
>                  if 'autotools' not in inherits:
>                      inherits.append('autotools')
>
>      for ac in configure_acs:
>          parse_configure_ac(ac)
>
> # check the *.pro files and get build requires from 'PKGCONFIG'
> def process_qmake_pro(pkgdir):
>      global pkgname
>
>      pro_files = []
>      depth = 0
>      clause = ''
>      pre_char = ''
>
>      if os.path.isfile(os.path.join(pkgdir, pkgname + '.pro')) and 'qmake2' not in inherits:
>          inherits.append('qmake2')
>      for root, dirnames, filenames in os.walk(pkgdir):
>          for f in filenames:
>              if re.match('.*\.pro', f):
>                  pro_files.append(os.path.join(pkgdir, f))
>
>      for pro in pro_files:
>          need_next_line = False
>          f = file(pro)
>          for line in f:
>              if line.startswith('#'):
>                  continue
>              line = re.sub('\n', '', line)
>              clause += line
>              if line.endswith('\\'):
>                  continue
>
>              clause = clause.strip()
>              match =  re.match('PKGCONFIG\s*=(.*)', clause)
>              if match:
>                  dep = match.group(1)
>                  push_buildreq(dep)
>              match = re.match('PKGCONFIG\s*\+=(.*)', clause)
>              if match:
>                  dep = match.group(1)
>                  dep = dep.strip()
>                  dep = re.sub('\\\\', '', dep)
>                  dep = re.sub('\s+', ' ', dep)
>                  items = dep.split(' ')
>                  for item in items:
>                      push_buildreq(item)
>
>              clause = ''
>
>          f.close()
>
> def parse_spec_file(tmpdir):
>      # find the spec file first
>      spec_file = ''
>      for item in os.listdir(tmpdir):
>          if re.match('.*\.spec', item):
>              spec_file = os.path.join(tmpdir, item)
>              break
>
>      if len(spec_file) == 0:
>          return 1
>
>      global pkgname, pkgversion, summary, description, depends
>      global src_uri, homepage
>
>      f = file(spec_file)
>      for line in f:
>          match = re.match('Summary\s*:(.*)', line)
>          if match:
>              if len(summary) == 0:
>                  summary = match.group(1).strip()
>              continue
>          match = re.match('Name\s*:(.*)', line)
>          if match:
>              pkgname = match.group(1).strip()
>              continue
>          match = re.match('Version\s*:(.*)', line)
>          if match:
>              version = match.group(1).strip()
>              if re.match('\d+(\.\d+)?', version):
>                  pkgversion = version
>              continue
>          match = re.match('License\s*:(.*)', line)
>          if match:
>              license.append(match.group(1).strip())
>              continue
>          match = re.match('URL(?i)\s*:(.*)', line)
>          if match:
>              homepage = match.group(1).strip()
>              continue
>          match = re.match('Source0*\s*:(.*)', line)
>          if match:
>              src_uri = match.group(1).strip()
>              src_uri = re.sub('\%\{name\}', pkgname, src_uri)
>              src_uri = re.sub('\%\{version\}', pkgversion, src_uri)
>              continue
>          match = re.match('BuildRequires\s*:(.*)', line)
>          if match:
>              deps = match.group(1).strip()
>              dpes = re.sub(',', ' ', deps)
>              deps = re.sub('\s+', ' ', deps)
>              for item in deps.split(' '):
>                  push_buildreq(item)
>
>          match = re.match('%description\s*$', line)
>          if match:
>              for l in f:
>                  if l.startswith('%'):
>                      break
>                  description += l
>              description = re.sub('\n', '', description)
>
>      f.close()
>
> def write_bbfile():
>      if len(pkgname) == 0 or len(pkgversion) == 0:
>          fail_quit("Can't get package name or version.")
>
>      content = ''
>      content += 'DESCRIPTION = "' + description + '"\n'
>      content += 'SUMMARY = "' + summary + '"\n'
>      if len(homepage)>  0:
>          content += 'HOMEPAGE = "' + homepage + '"\n'
>
>      content += 'LICENSE = "'
>      for lic in license:
>          content += lic + ' '
>      content += '"\n'
>      content += 'LIC_FILES_CHKSUM = "'
>      for key in license_files:
>          content += 'file://' + key + ';md5=' + license_files[key] + ' \\\n';
>      content += '\t\t"\n\n';
>
>      if len(depends)>  0:
>          content += 'DEPENDS = "'
>          for dep in depends:
>              content += dep + ' '
>          content += ' "\n'
>
>      content += 'PR = "r0"\n\n'
>
>      content += 'SRC_URI = "' + src_uri + '"\n'
>      content += 'SRC_URI[md5sum] = "' + src_uri_md5sum + '"\n'
>      content += 'SRC_URI[sha256sum] = "' + src_uri_sha256sum + '"\n'
>
>      if len(inherits)>  0:
>          content += '\ninherit '
>          for inherit in inherits:
>              content += inherit + ' '
>          content += '\n'
>
>      bb_filename = pkgname + '_' + pkgversion + '.bb'
>      bb_file = open(bb_filename, 'w')
>      bb_file.write(content)
>      bb_file.close()
>      print 'Create bb file : ', bb_filename
>
> def fail_quit(msg):
>      print msg
>      clean_up()
>      exit(1)
>
> def clean_up():
>      shutil.rmtree(tmpdir)
>      pass
>
> ##########################################################
>
> if __name__ == '__main__':
>      if (len(sys.argv)<  2):
>          usage()
>          exit(1)
>
>      src_uri = sys.argv[1]
>
>      guess_name_by_uri(src_uri)
>      if pkgname is None:
>          print "Can't get the package name."
>          exit(1)
>
>      ret = download(src_uri)
>      if ret != 0:
>          fail_quit('Download package failed. Make sure the SRC_URI is valid.')
>
>      if unpack_package() != 0:
>          fail_quit('Unpack package failed.')
>
>      if pkgtype == 'rpm':
>          parse_spec_file(tmpdir)
>
>      get_md5_sha256_sum()
>
>      pkgdir = ''
>      for subdir in os.listdir(tmpdir):
>          subdir = os.path.join(tmpdir, subdir)
>          if os.path.isdir(subdir):
>              pkgdir = subdir
>              break
>
>      if pkgtype == 'tar':
>          parse_spec_file(pkgdir)
>
>      guess_description(pkgdir)
>      process_configure(pkgdir)
>
>      setup_licenses()
>      guess_license(pkgdir)
>
>      process_configure_ac(pkgdir)
>      process_qmake_pro(pkgdir)
>
>      write_bbfile()
>      clean_up()
>
>
>
>
> _______________________________________________
> poky mailing list
> poky@yoctoproject.org
> https://lists.yoctoproject.org/listinfo/poky
>

Re: Please help to review the Yocto 1656: Recipe creation/import script

[Kang Kai]

On 2012年01月20日 00:33, Mark Hatle wrote:
> On 1/19/12 3:33 AM, Kang Kai wrote:
>> Hi Josh,
>>
>> The attachment is the the design document V2 and implement.
>>
>> The V2 document changes are:
>> 1 remove patch_set from argument list
>> 2 deb is not support right now
>>
>> Any comment is welcome.
>> And I will on vacation until Jan 30, so my reply may not in time.
>>
>> Thanks& Regards,
>> Kai
>>
>>
>>
>>
>> recipe_creation_bb.txt
>>
>>
>> Design document for Recipe creation/import script
>>
>> * Purpose
>>
>> The feature is from Yocto 1.2 Bug 1656. A script or similar system
>> that would allow you to give is an upstream URL, tarball, patch set,
>> package (SRPM, or debian style) and generate a recipe based on those
>> instructions.
>>
>> Name it with bitbake-createbb, and integrate into the build system
>> to use the build system infrastructure that similiar with 
>> bitbake-runtask.
>

Hi Mark,

Thanks for your comments.

> I'd suggest changing the name to recipe-createbb or something. Since 
> this isn't a "bitbake" program (or is it?)
How about name it with "create-recipe"?

>
>> * Usage:
>> bitbake-createbb<SRC_URI>
>>
>> * Design Flow
>> 1 download the source package from the SRC_URI
>> 2 get package name and version from package or tarball name.
>> 3 unpack to /tmp/bitbake-createbb-pid
>> 4 if tarball, get summary and description from freshmeat.org, README 
>> files and pkgconfig files;
>> otherwise get them from SRPM spec file or debian control file
>> 5 get license, license file and checksum from LICENSE/COPYING files 
>> or freshmeat.org.
>
> For license files, I would suggest we scan for common filenames:
>
> LICENSE, COPYING, COPYING.LIB, and COPYRIGHT
>
> If it doesn't match the known checksums, we tell the user and let them 
> manually figure out the license type and verify things are consisten, 
> but we should be able to fill out the license filename for them.
Ok, if can't get license from known checksums I'll left messages for 
user to check manually.

Regards,
Kai

>
> Otherwise, it all looks fine to me.
>
> --Mark
>
>> 6 get dependent packages
>> |-- get them by parse configure files
>> |-- just parse spec file
>> 8 remove downloaded files
>>
>>
>>
>>
>>
>> bitbake-createbb
>>
>>
>> #!/usr/bin/env python
>>
>> import os
>> import sys
>> import shutil
>> import re
>> import urllib
>> import sha, md5, hashlib
>>
>> tmpdir = "/tmp/bitbake_createbb.%s" % os.getpid()
>> bin_file_name = ''
>> pkgname = ''
>> pkgversion = ''
>> summary = ''
>> description = ''
>> homepage = ''
>> license = []
>> license_files = {}
>> licenses = {}
>> depends = []
>> src_uri = ''
>> src_uri_md5sum = ''
>> src_uri_sha256sum = ''
>> inherits = []
>>
>> # support 2 package types: tar and rpm
>> pkgtype = ""
>>
>> def usage():
>> print "This is the bitbake import bb file script. Usage:\n"
>> print "\tbitbake-createbb<SRC_URI> \n"
>>
>> def download(pkg_uri):
>> try:
>> shutil.rmtree(tmpdir)
>> except:
>> pass
>>
>> cwd = os.getcwd()
>> os.mkdir(tmpdir)
>> os.chdir(tmpdir)
>> print 'Downloading package: ', pkg_uri, ' to ', tmpdir
>> ret = os.system("wget --quiet " + pkg_uri)
>> os.chdir(cwd)
>>
>> return ret
>>
>> def guess_name_by_uri(uri):
>> global bin_file_name, pkgname, pkgversion, pkgtype
>>
>> elems = uri.rsplit('/', 1)
>> pkg = bin_file_name = elems[1]
>>
>> if pkg.endswith("tgz"):
>> pkg = re.sub("tgz$", "tar.gz", pkg)
>>
>> match = re.match('(.*?)\-([0-9\.\-\~]+)\.tar', pkg)
>> if match:
>> pkgtype = 'tar'
>> pkgname = match.group(1)
>> pkgversion = match.group(2)
>>
>> match = re.match('(.*?)\-([0-9\.\~]+).*?\.src.rpm', pkg)
>> if match:
>> pkgtype = 'rpm'
>> pkgname = match.group(1)
>> pkgversion = match.group(2)
>>
>> def get_md5_sha256_sum():
>> global src_uri_md5sum, src_uri_sha256sum
>>
>> curdir = os.getcwd()
>> os.chdir(tmpdir)
>>
>> f = file(bin_file_name, 'r')
>> content = f.read()
>> f.close()
>>
>> md_five = md5.new()
>> md_five.update(content)
>> src_uri_md5sum = md_five.hexdigest()
>>
>> sha_256 = hashlib.sha256()
>> sha_256.update(content)
>> src_uri_sha256sum = sha_256.hexdigest()
>>
>> os.chdir(curdir)
>>
>> def unpack_package():
>> global bin_file_name
>>
>> cwd = os.getcwd()
>> os.chdir(tmpdir)
>> if pkgtype == 'rpm':
>> cmdline = "rpm2cpio " + bin_file_name + " | cpio -id"
>> ret = os.system(cmdline)
>> if ret != 0:
>> return ret
>> for item in os.listdir('.'):
>> if re.match(pkgname + '([0-9\.\-\~]*)\.tar', item):
>> bin_file_name = item
>> break
>>
>> cmdline = "tar axf " + bin_file_name
>> ret = os.system(cmdline)
>>
>> os.chdir(cwd)
>> return ret
>>
>> # get the fall-back description when other way fail
>> # check homepage at same time
>> def guess_description_from_readme(readme):
>> global description, homepage
>> f = file(readme)
>> state = 0
>> desc = ''
>>
>> for line in f:
>> if state == 1 and re.match('^\n', line) and len(desc)> 80:
>> state = 2
>> if state == 0 and len(line)> 1:
>> state = 1
>> if state == 1:
>> desc += line
>>
>> match = re.search('(http\:\/\/.*$name.*\.org)', line)
>> if match:
>> url = match.group(1)
>> if re.search('bug') or len(homepage)> 1:
>> pass
>> else:
>> homepage = url
>>
>> f.close()
>> if (len(desc)> 4 and len(description)< 3):
>> description = desc
>>
>> def guess_description_from_freecode(pkgname):
>> global description
>> desc = ''
>> state = 0
>>
>> html = urllib.urlopen("http://freecode.com/projects/" + pkgname)
>> for line in html:
>> if state == 1:
>> desc += line
>> if state == 0 and re.search('\<div class\=\"project-detail\"\>', line):
>> state = 1
>> if state == 1 and re.search('\<\/p\>', line):
>> state = 2
>>
>> # deal the description
>> desc = re.sub('\<p\>', '', desc)
>> desc = re.sub('\<\/p\>', '', desc)
>> desc = re.sub('\r', '', desc)
>> desc = desc.strip()
>> if len(desc)> 10:
>> description = desc
>>
>> # get Summary from pkgconfig file
>> def guess_summary_from_pc(pc):
>> global summary
>>
>> fn = file(pc)
>> for line in fn:
>> match = re.match('Description:\s*(.*)', line)
>> if match and len(summary)< 2:
>> summary = match.group(1)
>> break
>>
>> summary = summary.strip()
>> fn.close()
>>
>> def guess_description(pkgdir):
>> global pkgname, description, summary
>>
>> readmes = []
>> pcs = []
>>
>> for subdir in os.listdir(pkgdir):
>> if re.match('^README$', subdir):
>> readmes.insert(0, os.path.join(pkgdir, subdir))
>> elif re.match('README.*', subdir):
>> readmes.append(os.path.join(pkgdir, subdir))
>> elif re.match('.*\.pc.*', subdir):
>> pcs.insert(0, os.path.join(pkgdir, subdir))
>> elif re.match(pkgname + '\.pc.*', subdir):
>> pcs.insert(0, os.path.join(pkgdir, subdir))
>> elif re.match('.*\.pc', subdir):
>> pcs.append(os.path.join(pkgdir, subdir))
>>
>> for readme in readmes:
>> guess_description_from_readme(os.path.join(pkgdir, readme))
>>
>> if (len(pkgname)> 2):
>> guess_description_from_freecode(pkgname)
>>
>> for pc in pcs:
>> guess_summary_from_pc(pc)
>>
>> # if didn't get summary, use first line of description
>> if len(summary)< 2:
>> summary = description
>> summary = re.sub("\n", " ", summary)
>> summary = re.sub("\s+", " ", summary)
>> match = re.match("(.*?)\.", summary)
>> summary = match.group(1)
>>
>> # the sha1sum values are from autospectacle
>> def setup_licenses():
>> licenses['06877624ea5c77efe3b7e39b0f909eda6e25a4ec'] = "GPLv2"
>> licenses["075d599585584bb0e4b526f5c40cb6b17e0da35a"] = "GPLv2"
>> licenses["10782dd732f42f49918c839e8a5e2894c508b079"] = "GPLv2"
>> licenses["2d29c273fda30310211bbf6a24127d589be09b6c"] = "GPLv2"
>> licenses["4df5d4b947cf4e63e675729dd3f168ba844483c7"] = "LGPLv2.1"
>> licenses["503df7650052cf38efde55e85f0fe363e59b9739"] = "GPLv2"
>> licenses["5405311284eab5ab51113f87c9bfac435c695bb9"] = "GPLv2"
>> licenses["5fb362ef1680e635fe5fb212b55eef4db9ead48f"] = "LGPLv2"
>> licenses["68c94ffc34f8ad2d7bfae3f5a6b996409211c1b1"] = "GPLv2"
>> licenses["66c77efd1cf9c70d4f982ea59487b2eeb6338e26"] = "LGPLv2.1"
>> licenses["74a8a6531a42e124df07ab5599aad63870fa0bd4"] = "GPLv2"
>> licenses["8088b44375ef05202c0fca4e9e82d47591563609"] = "LGPLv2.1"
>> licenses["8624bcdae55baeef00cd11d5dfcfa60f68710a02"] = "GPLv3"
>> licenses["8e57ffebd0ed4417edc22e3f404ea3664d7fed27"] = "MIT"
>> licenses["99b5245b4714b9b89e7584bfc88da64e2d315b81"] = "BSD"
>> licenses["aba8d76d0af67d57da3c3c321caa59f3d242386b"] = "MPLv1.1"
>> licenses["bf50bac24e7ec325dbb09c6b6c4dcc88a7d79e8f"] = "LGPLv2"
>> licenses["caeb68c46fa36651acf592771d09de7937926bb3"] = "LGPLv2.1"
>> licenses["dfac199a7539a404407098a2541b9482279f690d"] = "GPLv2"
>> licenses["e60c2e780886f95df9c9ee36992b8edabec00bcc"] = "LGPLv2.1"
>> licenses["c931aad3017d975b7f20666cde0953234a9efde3"] = "GPLv2"
>>
>> def guess_licenses_from_file(copying, relname):
>> global licenses, license
>>
>> sha1 = sha.new()
>>
>> fn = open(copying)
>> content = fn.read()
>> fn.close()
>>
>> sha1.update(content)
>> digest = sha1.hexdigest()
>>
>> if digest in licenses:
>> license.append(licenses[digest])
>> md_five = md5.new()
>> md_five.update(content)
>> license_files[relname] = md_five.hexdigest()
>>
>> def guess_licenses_from_freecode():
>> global license
>>
>> lic = ''
>> state = 0
>>
>> html = urllib.urlopen("http://freecode.com/projects/" + pkgname)
>> for line in html:
>> if state == 1:
>> lic += line
>> if state == 0 and re.search('<th><span>Licenses</span></th>', line):
>> state = 1
>> if state == 1 and re.search('</a>', line):
>> state = 2
>>
>> # deal the license
>> lic = lic.strip()
>> match = re.search('<a.*?>(.*?)</a>', lic)
>> if match:
>> license.append(match.group(1))
>>
>> def guess_license(pkgdir):
>> for item in os.listdir(pkgdir):
>> realpath = os.path.join(pkgdir, item)
>> if os.path.isfile(realpath) and (re.match('COPY.*', item) \
>> or re.match('LICENSE.*', item) or re.match('GPL.*', item)):
>> guess_licenses_from_file(realpath, item)
>>
>> if len(license) == 0:
>> guess_licenses_from_freecode()
>>
>> def process_configure(pkgdir):
>> if os.path.isfile(os.path.join(pkgdir, 'autogen.sh')):
>> os.system("cd " + pkgdir + " ; ./autogen.sh&> /dev/null");
>>
>> configure = os.path.join(pkgdir, 'configure')
>> if os.path.isfile(configure):
>> if 'autotools' not in inherits:
>> inherits.append('autotools')
>>
>> fn = open(configure)
>> for line in fn:
>> match = re.match('PACKAGE_NAME=\'(.*?)\'', line)
>> if match and len(pkgname) == 0:
>> pkgname = match.group(1)
>>
>> match = re.match('PACKAGE_TARNAME=\'(.*?)\'', line)
>> if match:
>> pkgname = match.group(1)
>>
>> match = re.match('PACKAGE_VERSION=\'(.*?)\'', line)
>> if match:
>> pkgversion = match.group(1)
>>
>> match = re.match('PACKAGE_URL=\'(.*?)\'', line)
>> if match:
>> homepage = match.group(1)
>>
>> fn.close()
>>
>> def push_buildreq(dep):
>> global depends
>>
>> # remove collateral ] ) etc damage in the string
>> dep = re.sub("\"", "", dep)
>> dep = re.sub("\)", "", dep)
>> dep = re.sub("\]", "", dep)
>> dep = re.sub("\[", "", dep)
>>
>> # first, undo the space packing
>> dep = re.sub(">=", ">= ", dep)
>> dep = re.sub("<=", "<= ", dep)
>>
>> items = dep.split(' ')
>> dep = items[0]
>>
>> # don't show configure variables, we can't deal with them
>> if re.search("\$", dep):
>> return
>> if re.search("AC_SUBST", dep):
>> return
>>
>> if dep not in depends:
>> depends.append(dep)
>>
>> def parse_configure_ac(ac_file):
>> depth = 0
>> clause = ""
>>
>> fac = file(ac_file)
>>
>> for line in fac:
>> line = line.strip()
>> i = 0
>> while i< len(line):
>> if line[i] == '(':
>> depth += 1
>> if line[i] == ')' and depth> 0:
>> depth -= 1
>> clause += line[i]
>> i += 1
>> if depth> 0:
>> continue
>>
>> # remove '\n'
>> clause = re.sub('\n', '', clause)
>> clause = clause.strip()
>>
>> match = re.match('PKG_CHECK_MODULES\((.*)\)', clause)
>> if match:
>> modules = match.group(1)
>> pkg = modules.split(',', 1)[1].strip()
>> match2 = re.match('\[(.*)\]', pkg)
>> if match2:
>> pkg = match2.group(1)
>>
>> # split the build dependencies
>> pkg = pkg.replace('\s+', ' ')
>> pkg = re.sub('\s>\s', '>', pkg)
>> pkg = re.sub('\s>=\s', '>=', pkg)
>> pkg = re.sub('\s=\s', '=', pkg)
>> pkg = re.sub('\s<=\s', '<=', pkg)
>> pkg = re.sub('\s<\s', '<', pkg)
>> pkglist = pkg.split(' ')
>> for dep in pkglist:
>> push_buildreq(dep)
>>
>> match = re.match('PKG_CHECK_EXISTS\((.*)\)', clause)
>> if match:
>> exists = match.group(1)
>> pkg = exists.split(',', 1)[0].strip()
>> match2 = re.match('\[(.*)\]', pkg)
>> if match2:
>> pkg = match2.group(1)
>> pkg = pkg.strip()
>>
>> pkg = re.sub('\s+', ' ', pkg)
>> pkg = re.sub('\s>\s', '>', pkg)
>> pkg = re.sub('\s>=\s', '>=', pkg)
>> pkg = re.sub('\s=\s', '=', pkg)
>> pkg = re.sub('\s<=\s', '<=', pkg)
>> pkg = re.sub('\s<\s', '<', pkg)
>> pkglist = pkg.split(' ')
>> for dep in pkglist:
>> push_buildreq(dep)
>>
>> # these items are from autospectacle.pl
>> if re.search('_PROG_INTLTOOL', clause):
>> push_buildreq("intltool")
>> if re.search('GETTEXT_PACKAGE', clause):
>> push_buildreq('gettext')
>> if re.search('GTK_DOC_CHECK', clause):
>> push_buildreq('gtk-doc')
>> if re.search('GNOME_DOC_INIT', clause):
>> push_buildreq('gnome-doc-utils')
>> if re.search('AM_GLIB_GNU_GETTEXT', clause):
>> push_buildreq('gettext')
>>
>> match = re.search('AC_INIT\((.*)\)', clause)
>> if match:
>> ac_init = match.group(1)
>> ac_init = ac_init.strip()
>> ac_init = re.sub('\s+', ' ', ac_init)
>> version = ac_init.split(',')[1].strip()
>> if re.match('\d+(\.\d+)*', version):
>> pkgversion = version
>>
>> match = re.search('AM_INIT_AUTOMAKE\((.*)\)', clause)
>> if match:
>> am_init = match.group(1)
>> am_init = re.sub('\s+', ' ', am_init)
>> items = am_init.split(',')
>> if len(items)>= 2:
>> ver = items[1]
>> ver = re.sub('\[', '', ver)
>> ver = re.sub('\]', '', ver)
>> if re.match('\d(\.\d+)*', ver):
>> pkgversion = ver
>> clause = ''
>> fac.close
>>
>> def process_configure_ac(pkgdir):
>> configure_acs = []
>> for root, dirnames, filenames in os.walk(pkgdir):
>> for f in filenames:
>> if re.match('.*\.ac', f):
>> configure_acs.append(os.path.join(root, f))
>> if 'autotools' not in inherits:
>> inherits.append('autotools')
>>
>> for ac in configure_acs:
>> parse_configure_ac(ac)
>>
>> # check the *.pro files and get build requires from 'PKGCONFIG'
>> def process_qmake_pro(pkgdir):
>> global pkgname
>>
>> pro_files = []
>> depth = 0
>> clause = ''
>> pre_char = ''
>>
>> if os.path.isfile(os.path.join(pkgdir, pkgname + '.pro')) and 
>> 'qmake2' not in inherits:
>> inherits.append('qmake2')
>> for root, dirnames, filenames in os.walk(pkgdir):
>> for f in filenames:
>> if re.match('.*\.pro', f):
>> pro_files.append(os.path.join(pkgdir, f))
>>
>> for pro in pro_files:
>> need_next_line = False
>> f = file(pro)
>> for line in f:
>> if line.startswith('#'):
>> continue
>> line = re.sub('\n', '', line)
>> clause += line
>> if line.endswith('\\'):
>> continue
>>
>> clause = clause.strip()
>> match = re.match('PKGCONFIG\s*=(.*)', clause)
>> if match:
>> dep = match.group(1)
>> push_buildreq(dep)
>> match = re.match('PKGCONFIG\s*\+=(.*)', clause)
>> if match:
>> dep = match.group(1)
>> dep = dep.strip()
>> dep = re.sub('\\\\', '', dep)
>> dep = re.sub('\s+', ' ', dep)
>> items = dep.split(' ')
>> for item in items:
>> push_buildreq(item)
>>
>> clause = ''
>>
>> f.close()
>>
>> def parse_spec_file(tmpdir):
>> # find the spec file first
>> spec_file = ''
>> for item in os.listdir(tmpdir):
>> if re.match('.*\.spec', item):
>> spec_file = os.path.join(tmpdir, item)
>> break
>>
>> if len(spec_file) == 0:
>> return 1
>>
>> global pkgname, pkgversion, summary, description, depends
>> global src_uri, homepage
>>
>> f = file(spec_file)
>> for line in f:
>> match = re.match('Summary\s*:(.*)', line)
>> if match:
>> if len(summary) == 0:
>> summary = match.group(1).strip()
>> continue
>> match = re.match('Name\s*:(.*)', line)
>> if match:
>> pkgname = match.group(1).strip()
>> continue
>> match = re.match('Version\s*:(.*)', line)
>> if match:
>> version = match.group(1).strip()
>> if re.match('\d+(\.\d+)?', version):
>> pkgversion = version
>> continue
>> match = re.match('License\s*:(.*)', line)
>> if match:
>> license.append(match.group(1).strip())
>> continue
>> match = re.match('URL(?i)\s*:(.*)', line)
>> if match:
>> homepage = match.group(1).strip()
>> continue
>> match = re.match('Source0*\s*:(.*)', line)
>> if match:
>> src_uri = match.group(1).strip()
>> src_uri = re.sub('\%\{name\}', pkgname, src_uri)
>> src_uri = re.sub('\%\{version\}', pkgversion, src_uri)
>> continue
>> match = re.match('BuildRequires\s*:(.*)', line)
>> if match:
>> deps = match.group(1).strip()
>> dpes = re.sub(',', ' ', deps)
>> deps = re.sub('\s+', ' ', deps)
>> for item in deps.split(' '):
>> push_buildreq(item)
>>
>> match = re.match('%description\s*$', line)
>> if match:
>> for l in f:
>> if l.startswith('%'):
>> break
>> description += l
>> description = re.sub('\n', '', description)
>>
>> f.close()
>>
>> def write_bbfile():
>> if len(pkgname) == 0 or len(pkgversion) == 0:
>> fail_quit("Can't get package name or version.")
>>
>> content = ''
>> content += 'DESCRIPTION = "' + description + '"\n'
>> content += 'SUMMARY = "' + summary + '"\n'
>> if len(homepage)> 0:
>> content += 'HOMEPAGE = "' + homepage + '"\n'
>>
>> content += 'LICENSE = "'
>> for lic in license:
>> content += lic + ' '
>> content += '"\n'
>> content += 'LIC_FILES_CHKSUM = "'
>> for key in license_files:
>> content += 'file://' + key + ';md5=' + license_files[key] + ' \\\n';
>> content += '\t\t"\n\n';
>>
>> if len(depends)> 0:
>> content += 'DEPENDS = "'
>> for dep in depends:
>> content += dep + ' '
>> content += ' "\n'
>>
>> content += 'PR = "r0"\n\n'
>>
>> content += 'SRC_URI = "' + src_uri + '"\n'
>> content += 'SRC_URI[md5sum] = "' + src_uri_md5sum + '"\n'
>> content += 'SRC_URI[sha256sum] = "' + src_uri_sha256sum + '"\n'
>>
>> if len(inherits)> 0:
>> content += '\ninherit '
>> for inherit in inherits:
>> content += inherit + ' '
>> content += '\n'
>>
>> bb_filename = pkgname + '_' + pkgversion + '.bb'
>> bb_file = open(bb_filename, 'w')
>> bb_file.write(content)
>> bb_file.close()
>> print 'Create bb file : ', bb_filename
>>
>> def fail_quit(msg):
>> print msg
>> clean_up()
>> exit(1)
>>
>> def clean_up():
>> shutil.rmtree(tmpdir)
>> pass
>>
>> ##########################################################
>>
>> if __name__ == '__main__':
>> if (len(sys.argv)< 2):
>> usage()
>> exit(1)
>>
>> src_uri = sys.argv[1]
>>
>> guess_name_by_uri(src_uri)
>> if pkgname is None:
>> print "Can't get the package name."
>> exit(1)
>>
>> ret = download(src_uri)
>> if ret != 0:
>> fail_quit('Download package failed. Make sure the SRC_URI is valid.')
>>
>> if unpack_package() != 0:
>> fail_quit('Unpack package failed.')
>>
>> if pkgtype == 'rpm':
>> parse_spec_file(tmpdir)
>>
>> get_md5_sha256_sum()
>>
>> pkgdir = ''
>> for subdir in os.listdir(tmpdir):
>> subdir = os.path.join(tmpdir, subdir)
>> if os.path.isdir(subdir):
>> pkgdir = subdir
>> break
>>
>> if pkgtype == 'tar':
>> parse_spec_file(pkgdir)
>>
>> guess_description(pkgdir)
>> process_configure(pkgdir)
>>
>> setup_licenses()
>> guess_license(pkgdir)
>>
>> process_configure_ac(pkgdir)
>> process_qmake_pro(pkgdir)
>>
>> write_bbfile()
>> clean_up()
>>
>>
>>
>>
>> _______________________________________________
>> poky mailing list
>> poky@yoctoproject.org
>> https://lists.yoctoproject.org/listinfo/poky
>>
>
> _______________________________________________
> poky mailing list
> poky@yoctoproject.org
> https://lists.yoctoproject.org/listinfo/poky

Re: Please help to review the Yocto 1656: Recipe creation/import script

[Joshua Lock]

On 19/01/12 01:33, Kang Kai wrote:
> Hi Josh,
>
> The attachment is the the design document V2 and implement.
>
> The V2 document changes are:
> 1 remove patch_set from argument list
> 2 deb is not support right now
>
> Any comment is welcome.
> And I will on vacation until Jan 30, so my reply may not in time.

This looks like a good start, Paul has offered some great feedback. In 
addition I'd add that we need a license header on the script, we should 
probably attribute autospectacle in the header too (Inspired by? Based on?).

I haven't had chance to do a thorough code review but will try and do so 
whilst you're on your break.

Cheers,
Joshua
-- 
Joshua Lock
         Yocto Project "Johannes factotum"
         Intel Open Source Technology Centre