* [RFC PATCH 01/21] bitbake/fetch2/git: Add backwards compatibility code for branch name handling
2011-01-24 14:32 [RFC PATCH 00/21] fetcher overhaul - api reorg and git optimization V2 Yu Ke
@ 2011-01-24 14:32 ` Yu Ke
2011-01-24 14:32 ` [RFC PATCH 02/21] bb.fetch2: add unpack method in fetcher Yu Ke
` (19 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Yu Ke @ 2011-01-24 14:32 UTC (permalink / raw)
To: poky
this patch has been applied for bitbake/fetch/git, so also apply it for bitbake/fetch/git
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
bitbake/lib/bb/fetch2/git.py | 82 ++++++++++++++++++++++++++++++++++++++++-
1 files changed, 80 insertions(+), 2 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index c621457..e85f1da 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -22,6 +22,7 @@ BitBake 'Fetch' git implementation
import os
import bb
+import bb.persist_data
from bb import data
from bb.fetch2 import Fetch
from bb.fetch2 import runfetchcmd
@@ -116,6 +117,7 @@ class Git(Fetch):
repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball)
+
coname = '%s' % (ud.tag)
codir = os.path.join(ud.clonedir, coname)
@@ -205,11 +207,19 @@ class Git(Fetch):
output = runfetchcmd("%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % (basecmd, tag), d, quiet=True)
return output.split()[0] != "0"
- def _revision_key(self, url, ud, d):
+ def _revision_key(self, url, ud, d, branch=False):
"""
Return a unique key for the url
"""
- return "git:" + ud.host + ud.path.replace('/', '.') + ud.branch
+ key = 'git:' + ud.host + ud.path.replace('/', '.')
+ if branch:
+ return key + ud.branch
+ else:
+ return key
+
+ def generate_revision_key(self, url, ud, d, branch=False):
+ key = self._revision_key(url, ud, d, branch)
+ return "%s-%s" % (key, bb.data.getVar("PN", d, True) or "")
def _latest_revision(self, url, ud, d):
"""
@@ -227,6 +237,74 @@ class Git(Fetch):
raise bb.fetch2.FetchError("Fetch command %s gave empty output\n" % (cmd))
return output.split()[0]
+ def latest_revision(self, url, ud, d):
+ """
+ Look in the cache for the latest revision, if not present ask the SCM.
+ """
+ persisted = bb.persist_data.persist(d)
+ revs = persisted['BB_URI_HEADREVS']
+
+ key = self.generate_revision_key(url, ud, d, branch=True)
+ rev = revs[key]
+ if rev is None:
+ # Compatibility with old key format, no branch included
+ oldkey = self.generate_revision_key(url, ud, d, branch=False)
+ rev = revs[oldkey]
+ if rev is not None:
+ del revs[oldkey]
+ else:
+ rev = self._latest_revision(url, ud, d)
+ revs[key] = rev
+
+ return str(rev)
+
+ def sortable_revision(self, url, ud, d):
+ """
+
+ """
+ pd = bb.persist_data.persist(d)
+ localcounts = pd['BB_URI_LOCALCOUNT']
+ key = self.generate_revision_key(url, ud, d, branch=True)
+ oldkey = self.generate_revision_key(url, ud, d, branch=False)
+
+ latest_rev = self._build_revision(url, ud, d)
+ last_rev = localcounts[key + '_rev']
+ if last_rev is None:
+ last_rev = localcounts[oldkey + '_rev']
+ if last_rev is not None:
+ del localcounts[oldkey + '_rev']
+ localcounts[key + '_rev'] = last_rev
+
+ uselocalcount = bb.data.getVar("BB_LOCALCOUNT_OVERRIDE", d, True) or False
+ count = None
+ if uselocalcount:
+ count = Fetch.localcount_internal_helper(ud, d)
+ if count is None:
+ count = localcounts[key + '_count']
+ if count is None:
+ count = localcounts[oldkey + '_count']
+ if count is not None:
+ del localcounts[oldkey + '_count']
+ localcounts[key + '_count'] = count
+
+ if last_rev == latest_rev:
+ return str(count + "+" + latest_rev)
+
+ buildindex_provided = hasattr(self, "_sortable_buildindex")
+ if buildindex_provided:
+ count = self._sortable_buildindex(url, ud, d, latest_rev)
+ if count is None:
+ count = "0"
+ elif uselocalcount or buildindex_provided:
+ count = str(count)
+ else:
+ count = str(int(count) + 1)
+
+ localcounts[key + '_rev'] = latest_rev
+ localcounts[key + '_count'] = count
+
+ return str(count + "+" + latest_rev)
+
def _build_revision(self, url, ud, d):
return ud.tag
--
1.7.0.4
^ permalink raw reply related [flat|nested] 22+ messages in thread* [RFC PATCH 02/21] bb.fetch2: add unpack method in fetcher
2011-01-24 14:32 [RFC PATCH 00/21] fetcher overhaul - api reorg and git optimization V2 Yu Ke
2011-01-24 14:32 ` [RFC PATCH 01/21] bitbake/fetch2/git: Add backwards compatibility code for branch name handling Yu Ke
@ 2011-01-24 14:32 ` Yu Ke
2011-01-24 14:32 ` [RFC PATCH 03/21] bb.fetch2: revise the Fetch.unpack API Yu Ke
` (18 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Yu Ke @ 2011-01-24 14:32 UTC (permalink / raw)
To: poky
copy exactly the base.bbclass:oe_unpack_file() to bb.fetch2 as the code base
Signed-off-by: Yu Ke <ke.yu@intel.com>
---
bitbake/lib/bb/fetch2/__init__.py | 84 +++++++++++++++++++++++++++++++++++++
1 files changed, 84 insertions(+), 0 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index 751e514..4b4ac47 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -259,6 +259,13 @@ def verify_checksum(u, ud, d):
ud.sha256_expected, sha256data)
raise FetchError("%s checksum mismatch." % u)
+def subprocess_setup():
+ import signal
+ # Python installs a SIGPIPE handler by default. This is usually not what
+ # non-Python subprocesses expect.
+ # SIGPIPE errors are known issues with gzip/bash
+ signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+
def go(d, urls = None):
"""
Fetch all urls
@@ -629,6 +636,83 @@ class Fetch(object):
"""
raise NoMethodError("Missing implementation for url")
+ def unpack(file, data, url = None):
+ import subprocess
+ if not url:
+ url = "file://%s" % file
+ dots = file.split(".")
+ if dots[-1] in ['gz', 'bz2', 'Z']:
+ efile = os.path.join(bb.data.getVar('WORKDIR', data, 1),os.path.basename('.'.join(dots[0:-1])))
+ else:
+ efile = file
+ cmd = None
+ if file.endswith('.tar'):
+ cmd = 'tar x --no-same-owner -f %s' % file
+ elif file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'):
+ cmd = 'tar xz --no-same-owner -f %s' % file
+ elif file.endswith('.tbz') or file.endswith('.tbz2') or file.endswith('.tar.bz2'):
+ cmd = 'bzip2 -dc %s | tar x --no-same-owner -f -' % file
+ elif file.endswith('.gz') or file.endswith('.Z') or file.endswith('.z'):
+ cmd = 'gzip -dc %s > %s' % (file, efile)
+ elif file.endswith('.bz2'):
+ cmd = 'bzip2 -dc %s > %s' % (file, efile)
+ elif file.endswith('.tar.xz'):
+ cmd = 'xz -dc %s | tar x --no-same-owner -f -' % file
+ elif file.endswith('.xz'):
+ cmd = 'xz -dc %s > %s' % (file, efile)
+ elif file.endswith('.zip') or file.endswith('.jar'):
+ cmd = 'unzip -q -o'
+ (type, host, path, user, pswd, parm) = bb.decodeurl(url)
+ if 'dos' in parm:
+ cmd = '%s -a' % cmd
+ cmd = "%s '%s'" % (cmd, file)
+ elif os.path.isdir(file):
+ filesdir = os.path.realpath(bb.data.getVar("FILESDIR", data, 1))
+ destdir = "."
+ if file[0:len(filesdir)] == filesdir:
+ destdir = file[len(filesdir):file.rfind('/')]
+ destdir = destdir.strip('/')
+ if len(destdir) < 1:
+ destdir = "."
+ elif not os.access("%s/%s" % (os.getcwd(), destdir), os.F_OK):
+ os.makedirs("%s/%s" % (os.getcwd(), destdir))
+ cmd = 'cp -pPR %s %s/%s/' % (file, os.getcwd(), destdir)
+ else:
+ (type, host, path, user, pswd, parm) = bb.decodeurl(url)
+ if not 'patch' in parm:
+ # The "destdir" handling was specifically done for FILESPATH
+ # items. So, only do so for file:// entries.
+ if type == "file" and path.find("/") != -1:
+ destdir = path.rsplit("/", 1)[0]
+ else:
+ destdir = "."
+ bb.mkdirhier("%s/%s" % (os.getcwd(), destdir))
+ cmd = 'cp %s %s/%s/' % (file, os.getcwd(), destdir)
+
+ if not cmd:
+ return True
+
+ dest = os.path.join(os.getcwd(), os.path.basename(file))
+ if os.path.exists(dest):
+ if os.path.samefile(file, dest):
+ return True
+
+ # Change to subdir before executing command
+ save_cwd = os.getcwd();
+ parm = bb.decodeurl(url)[5]
+ if 'subdir' in parm:
+ newdir = ("%s/%s" % (os.getcwd(), parm['subdir']))
+ bb.mkdirhier(newdir)
+ os.chdir(newdir)
+
+ cmd = "PATH=\"%s\" %s" % (bb.data.getVar('PATH', data, 1), cmd)
+ bb.note("Unpacking %s to %s/" % (file, os.getcwd()))
+ ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True)
+
+ os.chdir(save_cwd)
+
+ return ret == 0
+
def try_premirror(self, url, urldata, d):
"""
Should premirrors be used?
--
1.7.0.4
^ permalink raw reply related [flat|nested] 22+ messages in thread* [RFC PATCH 03/21] bb.fetch2: revise the Fetch.unpack API
2011-01-24 14:32 [RFC PATCH 00/21] fetcher overhaul - api reorg and git optimization V2 Yu Ke
2011-01-24 14:32 ` [RFC PATCH 01/21] bitbake/fetch2/git: Add backwards compatibility code for branch name handling Yu Ke
2011-01-24 14:32 ` [RFC PATCH 02/21] bb.fetch2: add unpack method in fetcher Yu Ke
@ 2011-01-24 14:32 ` Yu Ke
2011-01-24 14:32 ` [RFC PATCH 04/21] bb.fetch: add fetch version to distinguish bb.fetch and bb.fetch2 Yu Ke
` (17 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Yu Ke @ 2011-01-24 14:32 UTC (permalink / raw)
To: poky
change the unpack to use the urldata and rootdir parameter
- urldata is the FetchData instance
- rootdir is the dir to put the extracted source. the original unpack
use current dir (os.getcwd) as destination dir, which is not flexible
and error-prone (error will occur if caller not chdir to dest dir)
Signed-off-by: Yu Ke <ke.yu@intel.com>
---
bitbake/lib/bb/fetch2/__init__.py | 33 +++++++++++++++------------------
1 files changed, 15 insertions(+), 18 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index 4b4ac47..c5b5272 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -636,10 +636,9 @@ class Fetch(object):
"""
raise NoMethodError("Missing implementation for url")
- def unpack(file, data, url = None):
+ def unpack(self, urldata, rootdir, data):
import subprocess
- if not url:
- url = "file://%s" % file
+ file = urldata.localpath
dots = file.split(".")
if dots[-1] in ['gz', 'bz2', 'Z']:
efile = os.path.join(bb.data.getVar('WORKDIR', data, 1),os.path.basename('.'.join(dots[0:-1])))
@@ -662,8 +661,7 @@ class Fetch(object):
cmd = 'xz -dc %s > %s' % (file, efile)
elif file.endswith('.zip') or file.endswith('.jar'):
cmd = 'unzip -q -o'
- (type, host, path, user, pswd, parm) = bb.decodeurl(url)
- if 'dos' in parm:
+ if 'dos' in urldata.parm:
cmd = '%s -a' % cmd
cmd = "%s '%s'" % (cmd, file)
elif os.path.isdir(file):
@@ -674,34 +672,33 @@ class Fetch(object):
destdir = destdir.strip('/')
if len(destdir) < 1:
destdir = "."
- elif not os.access("%s/%s" % (os.getcwd(), destdir), os.F_OK):
- os.makedirs("%s/%s" % (os.getcwd(), destdir))
- cmd = 'cp -pPR %s %s/%s/' % (file, os.getcwd(), destdir)
+ elif not os.access("%s/%s" % (rootdir, destdir), os.F_OK):
+ os.makedirs("%s/%s" % (rootdir, destdir))
+ cmd = 'cp -pPR %s %s/%s/' % (file, rootdir, destdir)
else:
- (type, host, path, user, pswd, parm) = bb.decodeurl(url)
- if not 'patch' in parm:
+ if not 'patch' in urldata.parm:
# The "destdir" handling was specifically done for FILESPATH
# items. So, only do so for file:// entries.
- if type == "file" and path.find("/") != -1:
- destdir = path.rsplit("/", 1)[0]
+ if urldata.type == "file" and urldata.path.find("/") != -1:
+ destdir = urldata.path.rsplit("/", 1)[0]
else:
destdir = "."
- bb.mkdirhier("%s/%s" % (os.getcwd(), destdir))
- cmd = 'cp %s %s/%s/' % (file, os.getcwd(), destdir)
+ bb.mkdirhier("%s/%s" % (rootdir, destdir))
+ cmd = 'cp %s %s/%s/' % (file, rootdir, destdir)
if not cmd:
return True
- dest = os.path.join(os.getcwd(), os.path.basename(file))
+ dest = os.path.join(rootdir, os.path.basename(file))
if os.path.exists(dest):
if os.path.samefile(file, dest):
return True
# Change to subdir before executing command
save_cwd = os.getcwd();
- parm = bb.decodeurl(url)[5]
- if 'subdir' in parm:
- newdir = ("%s/%s" % (os.getcwd(), parm['subdir']))
+ os.chdir(rootdir)
+ if 'subdir' in urldata.parm:
+ newdir = ("%s/%s" % (rootdir, urldata.parm['subdir']))
bb.mkdirhier(newdir)
os.chdir(newdir)
--
1.7.0.4
^ permalink raw reply related [flat|nested] 22+ messages in thread* [RFC PATCH 04/21] bb.fetch: add fetch version to distinguish bb.fetch and bb.fetch2
2011-01-24 14:32 [RFC PATCH 00/21] fetcher overhaul - api reorg and git optimization V2 Yu Ke
` (2 preceding siblings ...)
2011-01-24 14:32 ` [RFC PATCH 03/21] bb.fetch2: revise the Fetch.unpack API Yu Ke
@ 2011-01-24 14:32 ` Yu Ke
2011-01-24 14:32 ` [RFC PATCH 05/21] base.bbclass: use bb.fetch2 unpack API Yu Ke
` (16 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Yu Ke @ 2011-01-24 14:32 UTC (permalink / raw)
To: poky
there is case that we need to distingush bb.fetch and bb.fetch2,
and use different API for bb.fetch and bb.fetch2. so it is necessary
to add version info for distinguish purpose
Signed-off-by: Yu Ke <ke.yu@intel.com>
---
bitbake/lib/bb/fetch/__init__.py | 2 ++
bitbake/lib/bb/fetch2/__init__.py | 2 ++
2 files changed, 4 insertions(+), 0 deletions(-)
diff --git a/bitbake/lib/bb/fetch/__init__.py b/bitbake/lib/bb/fetch/__init__.py
index 07eb77d..49fe502 100644
--- a/bitbake/lib/bb/fetch/__init__.py
+++ b/bitbake/lib/bb/fetch/__init__.py
@@ -32,6 +32,8 @@ import bb
from bb import data
from bb import persist_data
+__version__ = "1"
+
logger = logging.getLogger("BitBake.Fetch")
class MalformedUrl(Exception):
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index c5b5272..636178c 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -32,6 +32,8 @@ import bb
from bb import data
from bb import persist_data
+__version__ = "2"
+
logger = logging.getLogger("BitBake.Fetch")
class MalformedUrl(Exception):
--
1.7.0.4
^ permalink raw reply related [flat|nested] 22+ messages in thread* [RFC PATCH 05/21] base.bbclass: use bb.fetch2 unpack API
2011-01-24 14:32 [RFC PATCH 00/21] fetcher overhaul - api reorg and git optimization V2 Yu Ke
` (3 preceding siblings ...)
2011-01-24 14:32 ` [RFC PATCH 04/21] bb.fetch: add fetch version to distinguish bb.fetch and bb.fetch2 Yu Ke
@ 2011-01-24 14:32 ` Yu Ke
2011-01-24 14:32 ` [RFC PATCH 06/21] bb.fetch2: rename "go" with "download" to better reflect its functionality Yu Ke
` (15 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Yu Ke @ 2011-01-24 14:32 UTC (permalink / raw)
To: poky
Signed-off-by: Yu Ke <ke.yu@intel.com>
---
meta/classes/base.bbclass | 8 +++++++-
1 files changed, 7 insertions(+), 1 deletions(-)
diff --git a/meta/classes/base.bbclass b/meta/classes/base.bbclass
index f1ffb45..23b17f2 100644
--- a/meta/classes/base.bbclass
+++ b/meta/classes/base.bbclass
@@ -256,7 +256,13 @@ python base_do_unpack() {
continue
local = os.path.realpath(local)
lf = bb.utils.lockfile(urldata[url].lockfile)
- ret = oe_unpack_file(local, localdata, url)
+ if bb.fetch.__version__ == "1":
+ ret = oe_unpack_file(local, localdata, url)
+ else:
+ # use bb.fetch2 unpack API
+ ud = urldata[url]
+ rootdir = bb.data.getVar('WORKDIR', localdata, True)
+ ret = ud.method.unpack(ud, rootdir, localdata)
bb.utils.unlockfile(lf)
if not ret:
raise bb.build.FuncFailed("oe_unpack_file failed with return value %s" % ret)
--
1.7.0.4
^ permalink raw reply related [flat|nested] 22+ messages in thread* [RFC PATCH 06/21] bb.fetch2: rename "go" with "download" to better reflect its functionality
2011-01-24 14:32 [RFC PATCH 00/21] fetcher overhaul - api reorg and git optimization V2 Yu Ke
` (4 preceding siblings ...)
2011-01-24 14:32 ` [RFC PATCH 05/21] base.bbclass: use bb.fetch2 unpack API Yu Ke
@ 2011-01-24 14:32 ` Yu Ke
2011-01-24 14:32 ` [RFC PATCH 07/21] bbclasee: rename "go" with "download" Yu Ke
` (14 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Yu Ke @ 2011-01-24 14:32 UTC (permalink / raw)
To: poky
no functional change
Signed-off-by: Yu Ke <ke.yu@intel.com>
---
bitbake/lib/bb/fetch2/__init__.py | 8 ++++----
bitbake/lib/bb/fetch2/bzr.py | 2 +-
bitbake/lib/bb/fetch2/cvs.py | 2 +-
bitbake/lib/bb/fetch2/git.py | 6 +++---
bitbake/lib/bb/fetch2/hg.py | 2 +-
bitbake/lib/bb/fetch2/local.py | 2 +-
bitbake/lib/bb/fetch2/osc.py | 2 +-
bitbake/lib/bb/fetch2/perforce.py | 2 +-
bitbake/lib/bb/fetch2/repo.py | 2 +-
bitbake/lib/bb/fetch2/ssh.py | 2 +-
bitbake/lib/bb/fetch2/svk.py | 2 +-
bitbake/lib/bb/fetch2/svn.py | 2 +-
bitbake/lib/bb/fetch2/wget.py | 4 ++--
13 files changed, 19 insertions(+), 19 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index 636178c..ca48e57 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -268,7 +268,7 @@ def subprocess_setup():
# SIGPIPE errors are known issues with gzip/bash
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
-def go(d, urls = None):
+def download(d, urls = None):
"""
Fetch all urls
init must have previously been called
@@ -298,7 +298,7 @@ def go(d, urls = None):
if m.forcefetch(u, ud, d) or not localpath:
# Next try fetching from the original uri, u
try:
- m.go(u, ud, d)
+ m.download(u, ud, d)
localpath = ud.localpath
except FetchError:
# Remove any incomplete file
@@ -504,7 +504,7 @@ def try_mirrors(d, uri, mirrors, check = False, force = False):
if found:
return found
else:
- ud.method.go(newuri, ud, ld)
+ ud.method.download(newuri, ud, ld)
return ud.localpath
except (bb.fetch2.MissingParameterError,
bb.fetch2.FetchError,
@@ -631,7 +631,7 @@ class Fetch(object):
"""
return False
- def go(self, url, urldata, d):
+ def download(self, url, urldata, d):
"""
Fetch urls
Assumes localpath was called first
diff --git a/bitbake/lib/bb/fetch2/bzr.py b/bitbake/lib/bb/fetch2/bzr.py
index 97b042b..608ecc7 100644
--- a/bitbake/lib/bb/fetch2/bzr.py
+++ b/bitbake/lib/bb/fetch2/bzr.py
@@ -79,7 +79,7 @@ class Bzr(Fetch):
return bzrcmd
- def go(self, loc, ud, d):
+ def download(self, loc, ud, d):
"""Fetch url"""
if os.access(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir), '.bzr'), os.R_OK):
diff --git a/bitbake/lib/bb/fetch2/cvs.py b/bitbake/lib/bb/fetch2/cvs.py
index 1570cab..8e72090 100644
--- a/bitbake/lib/bb/fetch2/cvs.py
+++ b/bitbake/lib/bb/fetch2/cvs.py
@@ -72,7 +72,7 @@ class Cvs(Fetch):
return True
return False
- def go(self, loc, ud, d):
+ def download(self, loc, ud, d):
method = ud.parm.get('method', 'pserver')
localdir = ud.parm.get('localdir', ud.module)
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index e85f1da..c40a19c 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -107,7 +107,7 @@ class Git(Fetch):
return True
- def go(self, loc, ud, d):
+ def download(self, loc, ud, d):
"""Fetch url"""
if ud.user:
@@ -320,7 +320,7 @@ class Git(Fetch):
if not os.path.exists(ud.clonedir):
print("no repo")
- self.go(None, ud, d)
+ self.download(None, ud, d)
if not os.path.exists(ud.clonedir):
logger.error("GIT repository for %s doesn't exist in %s, cannot get sortable buildnumber, using old value", url, ud.clonedir)
return None
@@ -328,7 +328,7 @@ class Git(Fetch):
os.chdir(ud.clonedir)
if not self._contains_ref(rev, d):
- self.go(None, ud, d)
+ self.download(None, ud, d)
output = runfetchcmd("%s rev-list %s -- 2> /dev/null | wc -l" % (ud.basecmd, rev), d, quiet=True)
os.chdir(cwd)
diff --git a/bitbake/lib/bb/fetch2/hg.py b/bitbake/lib/bb/fetch2/hg.py
index 0ba8433..635ecbf 100644
--- a/bitbake/lib/bb/fetch2/hg.py
+++ b/bitbake/lib/bb/fetch2/hg.py
@@ -112,7 +112,7 @@ class Hg(Fetch):
return cmd
- def go(self, loc, ud, d):
+ def download(self, loc, ud, d):
"""Fetch url"""
logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")
diff --git a/bitbake/lib/bb/fetch2/local.py b/bitbake/lib/bb/fetch2/local.py
index bcb30df..89fbdf6 100644
--- a/bitbake/lib/bb/fetch2/local.py
+++ b/bitbake/lib/bb/fetch2/local.py
@@ -56,7 +56,7 @@ class Local(Fetch):
# We don't set localfile as for this fetcher the file is already local!
return newpath
- def go(self, url, urldata, d):
+ def download(self, url, urldata, d):
"""Fetch urls (no-op for Local method)"""
# no need to fetch local files, we'll deal with them in place.
return 1
diff --git a/bitbake/lib/bb/fetch2/osc.py b/bitbake/lib/bb/fetch2/osc.py
index 06ac5a9..619e2f1 100644
--- a/bitbake/lib/bb/fetch2/osc.py
+++ b/bitbake/lib/bb/fetch2/osc.py
@@ -79,7 +79,7 @@ class Osc(Fetch):
return osccmd
- def go(self, loc, ud, d):
+ def download(self, loc, ud, d):
"""
Fetch url
"""
diff --git a/bitbake/lib/bb/fetch2/perforce.py b/bitbake/lib/bb/fetch2/perforce.py
index 18b2781..bda0bb8 100644
--- a/bitbake/lib/bb/fetch2/perforce.py
+++ b/bitbake/lib/bb/fetch2/perforce.py
@@ -121,7 +121,7 @@ class Perforce(Fetch):
return os.path.join(data.getVar("DL_DIR", d, 1), ud.localfile)
- def go(self, loc, ud, d):
+ def download(self, loc, ud, d):
"""
Fetch urls
"""
diff --git a/bitbake/lib/bb/fetch2/repo.py b/bitbake/lib/bb/fetch2/repo.py
index 3330957..510ba46 100644
--- a/bitbake/lib/bb/fetch2/repo.py
+++ b/bitbake/lib/bb/fetch2/repo.py
@@ -55,7 +55,7 @@ class Repo(Fetch):
return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
- def go(self, loc, ud, d):
+ def download(self, loc, ud, d):
"""Fetch url"""
if os.access(os.path.join(data.getVar("DL_DIR", d, True), ud.localfile), os.R_OK):
diff --git a/bitbake/lib/bb/fetch2/ssh.py b/bitbake/lib/bb/fetch2/ssh.py
index 8b28322..78f55a6 100644
--- a/bitbake/lib/bb/fetch2/ssh.py
+++ b/bitbake/lib/bb/fetch2/ssh.py
@@ -74,7 +74,7 @@ class SSH(Fetch):
lpath = os.path.join(data.getVar('DL_DIR', d, True), host, os.path.basename(path))
return lpath
- def go(self, url, urldata, d):
+ def download(self, url, urldata, d):
dldir = data.getVar('DL_DIR', d, 1)
m = __pattern__.match(url)
diff --git a/bitbake/lib/bb/fetch2/svk.py b/bitbake/lib/bb/fetch2/svk.py
index 7990ff2..3bb4c38 100644
--- a/bitbake/lib/bb/fetch2/svk.py
+++ b/bitbake/lib/bb/fetch2/svk.py
@@ -57,7 +57,7 @@ class Svk(Fetch):
def forcefetch(self, url, ud, d):
return ud.date == "now"
- def go(self, loc, ud, d):
+ def download(self, loc, ud, d):
"""Fetch urls"""
svkroot = ud.host + ud.path
diff --git a/bitbake/lib/bb/fetch2/svn.py b/bitbake/lib/bb/fetch2/svn.py
index 1116795..547c04f 100644
--- a/bitbake/lib/bb/fetch2/svn.py
+++ b/bitbake/lib/bb/fetch2/svn.py
@@ -128,7 +128,7 @@ class Svn(Fetch):
return svncmd
- def go(self, loc, ud, d):
+ def download(self, loc, ud, d):
"""Fetch url"""
logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")
diff --git a/bitbake/lib/bb/fetch2/wget.py b/bitbake/lib/bb/fetch2/wget.py
index cf36cca..91cfafb 100644
--- a/bitbake/lib/bb/fetch2/wget.py
+++ b/bitbake/lib/bb/fetch2/wget.py
@@ -48,7 +48,7 @@ class Wget(Fetch):
return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
- def go(self, uri, ud, d, checkonly = False):
+ def download(self, uri, ud, d, checkonly = False):
"""Fetch urls"""
def fetch_uri(uri, ud, d):
@@ -90,4 +90,4 @@ class Wget(Fetch):
def checkstatus(self, uri, ud, d):
- return self.go(uri, ud, d, True)
+ return self.download(uri, ud, d, True)
--
1.7.0.4
^ permalink raw reply related [flat|nested] 22+ messages in thread* [RFC PATCH 07/21] bbclasee: rename "go" with "download"
2011-01-24 14:32 [RFC PATCH 00/21] fetcher overhaul - api reorg and git optimization V2 Yu Ke
` (5 preceding siblings ...)
2011-01-24 14:32 ` [RFC PATCH 06/21] bb.fetch2: rename "go" with "download" to better reflect its functionality Yu Ke
@ 2011-01-24 14:32 ` Yu Ke
2011-01-24 14:32 ` [RFC PATCH 08/21] git.py: split download to download() and build_mirror_data() Yu Ke
` (13 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Yu Ke @ 2011-01-24 14:32 UTC (permalink / raw)
To: poky
no functional change
Signed-off-by: Yu Ke <ke.yu@intel.com>
---
meta/classes/base.bbclass | 5 ++++-
meta/classes/sstate.bbclass | 5 ++++-
2 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/meta/classes/base.bbclass b/meta/classes/base.bbclass
index 23b17f2..d1d60f7 100644
--- a/meta/classes/base.bbclass
+++ b/meta/classes/base.bbclass
@@ -135,7 +135,10 @@ python base_do_fetch() {
raise bb.build.FuncFailed("Malformed URL: %s" % value)
try:
- bb.fetch.go(localdata)
+ if bb.fetch.__version__ == "1":
+ bb.fetch.go(localdata)
+ else:
+ bb.fetch.download(localdata)
except bb.fetch.MissingParameterError:
(type, value, traceback) = sys.exc_info()
raise bb.build.FuncFailed("Missing parameters: %s" % value)
diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index b6e6c92..fa184bc 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -319,7 +319,10 @@ def pstaging_fetch(sstatepkg, d):
# we will build the package
try:
bb.fetch.init([srcuri], localdata)
- bb.fetch.go(localdata, [srcuri])
+ if bb.fetch.__version__ == "1":
+ bb.fetch.go(localdata, [srcuri])
+ else:
+ bb.fetch.download(localdata, [srcuri])
# Need to optimise this, if using file:// urls, the fetcher just changes the local path
# For now work around by symlinking
localpath = bb.data.expand(bb.fetch.localpath(srcuri, localdata), localdata)
--
1.7.0.4
^ permalink raw reply related [flat|nested] 22+ messages in thread* [RFC PATCH 08/21] git.py: split download to download() and build_mirror_data()
2011-01-24 14:32 [RFC PATCH 00/21] fetcher overhaul - api reorg and git optimization V2 Yu Ke
` (6 preceding siblings ...)
2011-01-24 14:32 ` [RFC PATCH 07/21] bbclasee: rename "go" with "download" Yu Ke
@ 2011-01-24 14:32 ` Yu Ke
2011-01-24 14:32 ` [RFC PATCH 09/21] bb.fetch2: remove the obsolate Fetch.try_mirrors referrence Yu Ke
` (12 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Yu Ke @ 2011-01-24 14:32 UTC (permalink / raw)
To: poky
the download is to fetch the source from URL, the build_mirror_data is
to create the mirror tar ball. the original go() method mix them together,
it is more clean to split them.
Signed-off-by: Yu Ke <ke.yu@intel.com>
---
bitbake/lib/bb/fetch2/__init__.py | 4 ++++
bitbake/lib/bb/fetch2/git.py | 8 +++++---
2 files changed, 9 insertions(+), 3 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index ca48e57..466b62e 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -299,6 +299,8 @@ def download(d, urls = None):
# Next try fetching from the original uri, u
try:
m.download(u, ud, d)
+ if hasattr(m, "build_mirror_data"):
+ m.build_mirror_data(u, ud, d)
localpath = ud.localpath
except FetchError:
# Remove any incomplete file
@@ -505,6 +507,8 @@ def try_mirrors(d, uri, mirrors, check = False, force = False):
return found
else:
ud.method.download(newuri, ud, ld)
+ if hasattr(ud.method,"build_mirror_data"):
+ ud.method.build_mirror_data(newuri, ud, ld)
return ud.localpath
except (bb.fetch2.MissingParameterError,
bb.fetch2.FetchError,
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index c40a19c..690bb08 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -118,9 +118,6 @@ class Git(Fetch):
repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball)
- coname = '%s' % (ud.tag)
- codir = os.path.join(ud.clonedir, coname)
-
# If we have no existing clone and no mirror tarball, try and obtain one
if not os.path.exists(ud.clonedir) and not os.path.exists(repofile):
try:
@@ -151,7 +148,12 @@ class Git(Fetch):
runfetchcmd("%s prune-packed" % ud.basecmd, d)
runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d)
+ def build_mirror_data(self, url, ud, d):
# Generate a mirror tarball if needed
+ coname = '%s' % (ud.tag)
+ codir = os.path.join(ud.clonedir, coname)
+ repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball)
+
os.chdir(ud.clonedir)
mirror_tarballs = data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True)
if mirror_tarballs != "0" or 'fullclone' in ud.parm:
--
1.7.0.4
^ permalink raw reply related [flat|nested] 22+ messages in thread* [RFC PATCH 09/21] bb.fetch2: remove the obsolate Fetch.try_mirrors referrence
2011-01-24 14:32 [RFC PATCH 00/21] fetcher overhaul - api reorg and git optimization V2 Yu Ke
` (7 preceding siblings ...)
2011-01-24 14:32 ` [RFC PATCH 08/21] git.py: split download to download() and build_mirror_data() Yu Ke
@ 2011-01-24 14:32 ` Yu Ke
2011-01-24 14:32 ` [RFC PATCH 10/21] bb.fetch2: add git unpack Yu Ke
` (11 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Yu Ke @ 2011-01-24 14:32 UTC (permalink / raw)
To: poky
Fetch.try_mirrors is no longer exists, so the code is obsolate
Signed-off-by: Yu Ke <ke.yu@intel.com>
---
bitbake/lib/bb/fetch2/git.py | 8 --------
1 files changed, 0 insertions(+), 8 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index 690bb08..438756a 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -117,14 +117,6 @@ class Git(Fetch):
repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball)
-
- # If we have no existing clone and no mirror tarball, try and obtain one
- if not os.path.exists(ud.clonedir) and not os.path.exists(repofile):
- try:
- Fetch.try_mirrors(ud.mirrortarball)
- except:
- pass
-
# If the checkout doesn't exist and the mirror tarball does, extract it
if not os.path.exists(ud.clonedir) and os.path.exists(repofile):
bb.mkdirhier(ud.clonedir)
--
1.7.0.4
^ permalink raw reply related [flat|nested] 22+ messages in thread* [RFC PATCH 10/21] bb.fetch2: add git unpack
2011-01-24 14:32 [RFC PATCH 00/21] fetcher overhaul - api reorg and git optimization V2 Yu Ke
` (8 preceding siblings ...)
2011-01-24 14:32 ` [RFC PATCH 09/21] bb.fetch2: remove the obsolate Fetch.try_mirrors referrence Yu Ke
@ 2011-01-24 14:32 ` Yu Ke
2011-01-24 14:32 ` [RFC PATCH 11/21] git.py: remove the source tree tar ball Yu Ke
` (10 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Yu Ke @ 2011-01-24 14:32 UTC (permalink / raw)
To: poky
git download will clone git repo to local, and git unpack just do a clone with referrence to the original repo
Signed-off-by: Yu Ke <ke.yu@intel.com>
---
bitbake/lib/bb/fetch2/git.py | 16 ++++++++++++++++
1 files changed, 16 insertions(+), 0 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index 438756a..ca9d925 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -193,6 +193,22 @@ class Git(Fetch):
os.chdir(ud.clonedir)
bb.utils.prunedir(codir)
+ def unpack(self, ud, destdir, d):
+ """ unpack the downloaded src to destdir"""
+ subdir = ud.parm.get("subpath", "")
+ if subdir != "":
+ readpathspec = ":%s" % (subdir)
+ else:
+ readpathspec = ""
+
+ if 'fullclone' in ud.parm:
+ runfetchcmd("cp -pPR %s %s" % (os.path.join(ud.clonedir, ".git"), destdir), d)
+ else:
+ os.chdir(ud.clonedir)
+ runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.tag, readpathspec), d)
+ runfetchcmd("%s checkout-index -q -f --prefix=%s -a" % (ud.basecmd, os.path.join(destdir,"git/")), d)
+ return True
+
def supports_srcrev(self):
return True
--
1.7.0.4
^ permalink raw reply related [flat|nested] 22+ messages in thread* [RFC PATCH 11/21] git.py: remove the source tree tar ball
2011-01-24 14:32 [RFC PATCH 00/21] fetcher overhaul - api reorg and git optimization V2 Yu Ke
` (9 preceding siblings ...)
2011-01-24 14:32 ` [RFC PATCH 10/21] bb.fetch2: add git unpack Yu Ke
@ 2011-01-24 14:32 ` Yu Ke
2011-01-24 14:32 ` [RFC PATCH 12/21] bb.fetch2: add "BB_NO_NETWORK" option Yu Ke
` (9 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Yu Ke @ 2011-01-24 14:32 UTC (permalink / raw)
To: poky
we already create repo tar ball, so no need to create the source tree tar ball
Signed-off-by: Yu Ke <ke.yu@intel.com>
---
bitbake/lib/bb/fetch2/git.py | 62 +++---------------------------------------
1 files changed, 4 insertions(+), 58 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index ca9d925..acb01c6 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -61,24 +61,14 @@ class Git(Fetch):
ud.clonedir = os.path.join(data.expand('${GITDIR}', d), gitsrcname)
ud.basecmd = data.getVar("FETCHCMD_git", d, True) or "git"
+ ud.repochanged = False
def localpath(self, url, ud, d):
ud.tag = ud.revision
if not ud.tag or ud.tag == "master":
ud.tag = self.latest_revision(url, ud, d)
- subdir = ud.parm.get("subpath", "")
- if subdir != "":
- if subdir.endswith("/"):
- subdir = subdir[:-1]
- subdirpath = os.path.join(ud.path, subdir);
- else:
- subdirpath = ud.path;
-
- if 'fullclone' in ud.parm:
- ud.localfile = ud.mirrortarball
- else:
- ud.localfile = data.expand('git_%s%s_%s.tar.gz' % (ud.host, subdirpath.replace('/', '.'), ud.tag), d)
+ ud.localfile = ud.mirrortarball
if 'noclone' in ud.parm:
ud.localfile = None
@@ -91,8 +81,6 @@ class Git(Fetch):
return True
if 'noclone' in ud.parm:
return False
- if os.path.exists(ud.localpath):
- return False
if not self._contains_ref(ud.tag, d):
return True
return False
@@ -139,60 +127,18 @@ class Git(Fetch):
runfetchcmd("%s fetch --tags %s://%s%s%s" % (ud.basecmd, ud.proto, username, ud.host, ud.path), d)
runfetchcmd("%s prune-packed" % ud.basecmd, d)
runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d)
+ ud.repochanged = True
def build_mirror_data(self, url, ud, d):
# Generate a mirror tarball if needed
- coname = '%s' % (ud.tag)
- codir = os.path.join(ud.clonedir, coname)
repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball)
os.chdir(ud.clonedir)
mirror_tarballs = data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True)
- if mirror_tarballs != "0" or 'fullclone' in ud.parm:
+ if mirror_tarballs != "0" or 'fullclone' in ud.parm or ud.repochanged:
logger.info("Creating tarball of git repository")
runfetchcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ), d)
- if 'fullclone' in ud.parm:
- return
-
- if os.path.exists(codir):
- bb.utils.prunedir(codir)
-
- subdir = ud.parm.get("subpath", "")
- if subdir != "":
- if subdir.endswith("/"):
- subdirbase = os.path.basename(subdir[:-1])
- else:
- subdirbase = os.path.basename(subdir)
- else:
- subdirbase = ""
-
- if subdir != "":
- readpathspec = ":%s" % (subdir)
- codir = os.path.join(codir, "git")
- coprefix = os.path.join(codir, subdirbase, "")
- else:
- readpathspec = ""
- coprefix = os.path.join(codir, "git", "")
-
- scmdata = ud.parm.get("scmdata", "")
- if scmdata == "keep":
- runfetchcmd("%s clone -n %s %s" % (ud.basecmd, ud.clonedir, coprefix), d)
- os.chdir(coprefix)
- runfetchcmd("%s checkout -q -f %s%s" % (ud.basecmd, ud.tag, readpathspec), d)
- else:
- bb.mkdirhier(codir)
- os.chdir(ud.clonedir)
- runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.tag, readpathspec), d)
- runfetchcmd("%s checkout-index -q -f --prefix=%s -a" % (ud.basecmd, coprefix), d)
-
- os.chdir(codir)
- logger.info("Creating tarball of git checkout")
- runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.join(".", "*") ), d)
-
- os.chdir(ud.clonedir)
- bb.utils.prunedir(codir)
-
def unpack(self, ud, destdir, d):
""" unpack the downloaded src to destdir"""
subdir = ud.parm.get("subpath", "")
--
1.7.0.4
^ permalink raw reply related [flat|nested] 22+ messages in thread* [RFC PATCH 12/21] bb.fetch2: add "BB_NO_NETWORK" option
2011-01-24 14:32 [RFC PATCH 00/21] fetcher overhaul - api reorg and git optimization V2 Yu Ke
` (10 preceding siblings ...)
2011-01-24 14:32 ` [RFC PATCH 11/21] git.py: remove the source tree tar ball Yu Ke
@ 2011-01-24 14:32 ` Yu Ke
2011-01-24 14:32 ` [RFC PATCH 13/21] git.py: add no network access check Yu Ke
` (8 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Yu Ke @ 2011-01-24 14:32 UTC (permalink / raw)
To: poky
Sometime user want a purely local fetching, i.e. using local mirror without
any remote netowrk access. BB_NO_NETWORK option is introduced for this purpose
check_network_access() is the guard for BB_NO_NETWOKR option. it should be
put in any place that fetcher use network access
Signed-off-by: Yu Ke <ke.yu@intel.com>
---
bitbake/lib/bb/fetch2/__init__.py | 10 ++++++++++
1 files changed, 10 insertions(+), 0 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index 466b62e..ad94231 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -474,6 +474,16 @@ def runfetchcmd(cmd, d, quiet = False):
return output
+def check_network_access(d, info = ""):
+ """
+ log remote network access, and error if BB_NO_NETWORK is set
+ """
+ if bb.data.getVar("BB_NO_NETWORK", d, True) == "1":
+ bb.error("BB_NO_NETWORK is set, but fetcher access netowrk with [%s] " % info)
+ raise FetchError("BB_NO_NETWORK violation")
+ else:
+ bb.note("Fetcher access netowrk with [%s]" % info)
+
def try_mirrors(d, uri, mirrors, check = False, force = False):
"""
Try to use a mirrored version of the sources.
--
1.7.0.4
^ permalink raw reply related [flat|nested] 22+ messages in thread* [RFC PATCH 13/21] git.py: add no network access check
2011-01-24 14:32 [RFC PATCH 00/21] fetcher overhaul - api reorg and git optimization V2 Yu Ke
` (11 preceding siblings ...)
2011-01-24 14:32 ` [RFC PATCH 12/21] bb.fetch2: add "BB_NO_NETWORK" option Yu Ke
@ 2011-01-24 14:32 ` Yu Ke
2011-01-24 14:33 ` [RFC PATCH 14/21] wget.py: add no network check Yu Ke
` (7 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Yu Ke @ 2011-01-24 14:32 UTC (permalink / raw)
To: poky
Signed-off-by: Yu Ke <ke.yu@intel.com>
---
bitbake/lib/bb/fetch2/git.py | 3 +++
1 files changed, 3 insertions(+), 0 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index acb01c6..8f574b7 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -113,12 +113,14 @@ class Git(Fetch):
# If the repo still doesn't exist, fallback to cloning it
if not os.path.exists(ud.clonedir):
+ bb.fetch2.check_network_access(d, "git clone %s%s" % (ud.host, ud.path))
runfetchcmd("%s clone -n %s://%s%s%s %s" % (ud.basecmd, ud.proto, username, ud.host, ud.path, ud.clonedir), d)
os.chdir(ud.clonedir)
# Update the checkout if needed
if not self._contains_ref(ud.tag, d) or 'fullclone' in ud.parm:
# Remove all but the .git directory
+ bb.fetch2.check_network_access(d, "git fetch %s%s" %(ud.host, ud.path))
runfetchcmd("rm * -Rf", d)
if 'fullclone' in ud.parm:
runfetchcmd("%s fetch --all" % (ud.basecmd), d)
@@ -186,6 +188,7 @@ class Git(Fetch):
else:
username = ""
+ bb.fetch2.check_network_access(d, "git ls-remote %s%s %s" % (ud.host, ud.path, ud.branch))
basecmd = data.getVar("FETCHCMD_git", d, True) or "git"
cmd = "%s ls-remote %s://%s%s%s %s" % (basecmd, ud.proto, username, ud.host, ud.path, ud.branch)
output = runfetchcmd(cmd, d, True)
--
1.7.0.4
^ permalink raw reply related [flat|nested] 22+ messages in thread* [RFC PATCH 14/21] wget.py: add no network check
2011-01-24 14:32 [RFC PATCH 00/21] fetcher overhaul - api reorg and git optimization V2 Yu Ke
` (12 preceding siblings ...)
2011-01-24 14:32 ` [RFC PATCH 13/21] git.py: add no network access check Yu Ke
@ 2011-01-24 14:33 ` Yu Ke
2011-01-24 14:33 ` [RFC PATCH 15/21] svn.py: add no network access check Yu Ke
` (6 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Yu Ke @ 2011-01-24 14:33 UTC (permalink / raw)
To: poky
Signed-off-by: Yu Ke <ke.yu@intel.com>
---
bitbake/lib/bb/fetch2/wget.py | 1 +
1 files changed, 1 insertions(+), 0 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/wget.py b/bitbake/lib/bb/fetch2/wget.py
index 91cfafb..88cb8c7 100644
--- a/bitbake/lib/bb/fetch2/wget.py
+++ b/bitbake/lib/bb/fetch2/wget.py
@@ -69,6 +69,7 @@ class Wget(Fetch):
fetchcmd = fetchcmd.replace("${FILE}", ud.basename)
logger.info("fetch " + uri)
logger.debug(2, "executing " + fetchcmd)
+ bb.fetch2.check_network_access(d, fetchcmd)
runfetchcmd(fetchcmd, d)
# Sanity check since wget can pretend it succeed when it didn't
--
1.7.0.4
^ permalink raw reply related [flat|nested] 22+ messages in thread* [RFC PATCH 15/21] svn.py: add no network access check
2011-01-24 14:32 [RFC PATCH 00/21] fetcher overhaul - api reorg and git optimization V2 Yu Ke
` (13 preceding siblings ...)
2011-01-24 14:33 ` [RFC PATCH 14/21] wget.py: add no network check Yu Ke
@ 2011-01-24 14:33 ` Yu Ke
2011-01-24 14:33 ` [RFC PATCH 16/21] cvs.py: " Yu Ke
` (5 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Yu Ke @ 2011-01-24 14:33 UTC (permalink / raw)
To: poky
Signed-off-by: Yu Ke <ke.yu@intel.com>
---
bitbake/lib/bb/fetch2/svn.py | 4 +++-
1 files changed, 3 insertions(+), 1 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/svn.py b/bitbake/lib/bb/fetch2/svn.py
index 547c04f..8d76802 100644
--- a/bitbake/lib/bb/fetch2/svn.py
+++ b/bitbake/lib/bb/fetch2/svn.py
@@ -139,6 +139,7 @@ class Svn(Fetch):
# update sources there
os.chdir(ud.moddir)
logger.debug(1, "Running %s", svnupdatecmd)
+ bb.fetch2.check_network_access(d, svnupdatecmd)
runfetchcmd(svnupdatecmd, d)
else:
svnfetchcmd = self._buildsvncommand(ud, d, "fetch")
@@ -147,6 +148,7 @@ class Svn(Fetch):
bb.mkdirhier(ud.pkgdir)
os.chdir(ud.pkgdir)
logger.debug(1, "Running %s", svnfetchcmd)
+ bb.fetch2.check_network_access(d, svnfetchcmd)
runfetchcmd(svnfetchcmd, d)
scmdata = ud.parm.get("scmdata", "")
@@ -180,7 +182,7 @@ class Svn(Fetch):
"""
Return the latest upstream revision number
"""
- logger.debug(2, "SVN fetcher hitting network for %s", url)
+ bb.fetch2.check_network_access(d, self._buildsvncommand(ud, d, "info"))
output = runfetchcmd("LANG=C LC_ALL=C " + self._buildsvncommand(ud, d, "info"), d, True)
--
1.7.0.4
^ permalink raw reply related [flat|nested] 22+ messages in thread* [RFC PATCH 16/21] cvs.py: add no network access check
2011-01-24 14:32 [RFC PATCH 00/21] fetcher overhaul - api reorg and git optimization V2 Yu Ke
` (14 preceding siblings ...)
2011-01-24 14:33 ` [RFC PATCH 15/21] svn.py: add no network access check Yu Ke
@ 2011-01-24 14:33 ` Yu Ke
2011-01-24 14:33 ` [RFC PATCH 17/21] hg.py: " Yu Ke
` (4 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Yu Ke @ 2011-01-24 14:33 UTC (permalink / raw)
To: poky
Signed-off-by: Yu Ke <ke.yu@intel.com>
---
bitbake/lib/bb/fetch2/cvs.py | 2 ++
1 files changed, 2 insertions(+), 0 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/cvs.py b/bitbake/lib/bb/fetch2/cvs.py
index 8e72090..4915e74 100644
--- a/bitbake/lib/bb/fetch2/cvs.py
+++ b/bitbake/lib/bb/fetch2/cvs.py
@@ -131,6 +131,7 @@ class Cvs(Fetch):
moddir = os.path.join(pkgdir, localdir)
if os.access(os.path.join(moddir, 'CVS'), os.R_OK):
logger.info("Update " + loc)
+ bb.fetch2.check_network_access(d, cvsupdatecmd)
# update sources there
os.chdir(moddir)
myret = os.system(cvsupdatecmd)
@@ -140,6 +141,7 @@ class Cvs(Fetch):
bb.mkdirhier(pkgdir)
os.chdir(pkgdir)
logger.debug(1, "Running %s", cvscmd)
+ bb.fetch2.check_network_access(d, cvscmd)
myret = os.system(cvscmd)
if myret != 0 or not os.access(moddir, os.R_OK):
--
1.7.0.4
^ permalink raw reply related [flat|nested] 22+ messages in thread* [RFC PATCH 17/21] hg.py: add no network access check
2011-01-24 14:32 [RFC PATCH 00/21] fetcher overhaul - api reorg and git optimization V2 Yu Ke
` (15 preceding siblings ...)
2011-01-24 14:33 ` [RFC PATCH 16/21] cvs.py: " Yu Ke
@ 2011-01-24 14:33 ` Yu Ke
2011-01-24 14:33 ` [RFC PATCH 18/21] bzr.py: " Yu Ke
` (3 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Yu Ke @ 2011-01-24 14:33 UTC (permalink / raw)
To: poky
Signed-off-by: Yu Ke <ke.yu@intel.com>
---
bitbake/lib/bb/fetch2/hg.py | 3 +++
1 files changed, 3 insertions(+), 0 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/hg.py b/bitbake/lib/bb/fetch2/hg.py
index 635ecbf..4ba28c7 100644
--- a/bitbake/lib/bb/fetch2/hg.py
+++ b/bitbake/lib/bb/fetch2/hg.py
@@ -123,6 +123,7 @@ class Hg(Fetch):
# update sources there
os.chdir(ud.moddir)
logger.debug(1, "Running %s", updatecmd)
+ bb.fetch2.check_network_access(d, updatecmd)
runfetchcmd(updatecmd, d)
else:
@@ -132,6 +133,7 @@ class Hg(Fetch):
bb.mkdirhier(ud.pkgdir)
os.chdir(ud.pkgdir)
logger.debug(1, "Running %s", fetchcmd)
+ bb.fetch2.check_network_access(d, fetchcmd)
runfetchcmd(fetchcmd, d)
# Even when we clone (fetch), we still need to update as hg's clone
@@ -165,6 +167,7 @@ class Hg(Fetch):
"""
Compute tip revision for the url
"""
+ bb.fetch2.check_network_access(d, self._buildhgcommand(ud, d, "info"))
output = runfetchcmd(self._buildhgcommand(ud, d, "info"), d)
return output.strip()
--
1.7.0.4
^ permalink raw reply related [flat|nested] 22+ messages in thread* [RFC PATCH 18/21] bzr.py: add no network access check
2011-01-24 14:32 [RFC PATCH 00/21] fetcher overhaul - api reorg and git optimization V2 Yu Ke
` (16 preceding siblings ...)
2011-01-24 14:33 ` [RFC PATCH 17/21] hg.py: " Yu Ke
@ 2011-01-24 14:33 ` Yu Ke
2011-01-24 14:33 ` [RFC PATCH 19/21] repo.py: " Yu Ke
` (2 subsequent siblings)
20 siblings, 0 replies; 22+ messages in thread
From: Yu Ke @ 2011-01-24 14:33 UTC (permalink / raw)
To: poky
Signed-off-by: Yu Ke <ke.yu@intel.com>
---
bitbake/lib/bb/fetch2/bzr.py | 4 ++++
1 files changed, 4 insertions(+), 0 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/bzr.py b/bitbake/lib/bb/fetch2/bzr.py
index 608ecc7..6e1970b 100644
--- a/bitbake/lib/bb/fetch2/bzr.py
+++ b/bitbake/lib/bb/fetch2/bzr.py
@@ -85,11 +85,13 @@ class Bzr(Fetch):
if os.access(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir), '.bzr'), os.R_OK):
bzrcmd = self._buildbzrcommand(ud, d, "update")
logger.debug(1, "BZR Update %s", loc)
+ bb.fetch2.check_network_access(d, bzrcmd)
os.chdir(os.path.join (ud.pkgdir, os.path.basename(ud.path)))
runfetchcmd(bzrcmd, d)
else:
bb.utils.remove(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir)), True)
bzrcmd = self._buildbzrcommand(ud, d, "fetch")
+ bb.fetch2.check_network_access(d, bzrcmd)
logger.debug(1, "BZR Checkout %s", loc)
bb.mkdirhier(ud.pkgdir)
os.chdir(ud.pkgdir)
@@ -130,6 +132,8 @@ class Bzr(Fetch):
"""
logger.debug(2, "BZR fetcher hitting network for %s", url)
+ bb.fetch2.check_network_access(d, self._buildbzrcommand(ud, d, "revno"))
+
output = runfetchcmd(self._buildbzrcommand(ud, d, "revno"), d, True)
return output.strip()
--
1.7.0.4
^ permalink raw reply related [flat|nested] 22+ messages in thread* [RFC PATCH 19/21] repo.py: add no network access check
2011-01-24 14:32 [RFC PATCH 00/21] fetcher overhaul - api reorg and git optimization V2 Yu Ke
` (17 preceding siblings ...)
2011-01-24 14:33 ` [RFC PATCH 18/21] bzr.py: " Yu Ke
@ 2011-01-24 14:33 ` Yu Ke
2011-01-24 14:33 ` [RFC PATCH 20/21] osc.py: " Yu Ke
2011-01-24 14:33 ` [RFC PATCH 21/21] ssh.py: " Yu Ke
20 siblings, 0 replies; 22+ messages in thread
From: Yu Ke @ 2011-01-24 14:33 UTC (permalink / raw)
To: poky
Signed-off-by: Yu Ke <ke.yu@intel.com>
---
bitbake/lib/bb/fetch2/repo.py | 2 ++
1 files changed, 2 insertions(+), 0 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/repo.py b/bitbake/lib/bb/fetch2/repo.py
index 510ba46..c80fe5d 100644
--- a/bitbake/lib/bb/fetch2/repo.py
+++ b/bitbake/lib/bb/fetch2/repo.py
@@ -74,8 +74,10 @@ class Repo(Fetch):
bb.mkdirhier(os.path.join(codir, "repo"))
os.chdir(os.path.join(codir, "repo"))
if not os.path.exists(os.path.join(codir, "repo", ".repo")):
+ bb.fetch2.check_network_access(d, "repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path))
runfetchcmd("repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), d)
+ bb.fetch2.check_network_access(d, "repo sync %s" % ud.url)
runfetchcmd("repo sync", d)
os.chdir(codir)
--
1.7.0.4
^ permalink raw reply related [flat|nested] 22+ messages in thread* [RFC PATCH 20/21] osc.py: add no network access check
2011-01-24 14:32 [RFC PATCH 00/21] fetcher overhaul - api reorg and git optimization V2 Yu Ke
` (18 preceding siblings ...)
2011-01-24 14:33 ` [RFC PATCH 19/21] repo.py: " Yu Ke
@ 2011-01-24 14:33 ` Yu Ke
2011-01-24 14:33 ` [RFC PATCH 21/21] ssh.py: " Yu Ke
20 siblings, 0 replies; 22+ messages in thread
From: Yu Ke @ 2011-01-24 14:33 UTC (permalink / raw)
To: poky
Signed-off-by: Yu Ke <ke.yu@intel.com>
---
bitbake/lib/bb/fetch2/osc.py | 2 ++
1 files changed, 2 insertions(+), 0 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/osc.py b/bitbake/lib/bb/fetch2/osc.py
index 619e2f1..25dcb7b 100644
--- a/bitbake/lib/bb/fetch2/osc.py
+++ b/bitbake/lib/bb/fetch2/osc.py
@@ -92,6 +92,7 @@ class Osc(Fetch):
# update sources there
os.chdir(ud.moddir)
logger.debug(1, "Running %s", oscupdatecmd)
+ bb.fetch2.check_network_access(d, oscupdatecmd)
runfetchcmd(oscupdatecmd, d)
else:
oscfetchcmd = self._buildosccommand(ud, d, "fetch")
@@ -100,6 +101,7 @@ class Osc(Fetch):
bb.mkdirhier(ud.pkgdir)
os.chdir(ud.pkgdir)
logger.debug(1, "Running %s", oscfetchcmd)
+ bb.fetch2.check_network_access(d, oscfetchcmd)
runfetchcmd(oscfetchcmd, d)
os.chdir(os.path.join(ud.pkgdir + ud.path))
--
1.7.0.4
^ permalink raw reply related [flat|nested] 22+ messages in thread* [RFC PATCH 21/21] ssh.py: add no network access check
2011-01-24 14:32 [RFC PATCH 00/21] fetcher overhaul - api reorg and git optimization V2 Yu Ke
` (19 preceding siblings ...)
2011-01-24 14:33 ` [RFC PATCH 20/21] osc.py: " Yu Ke
@ 2011-01-24 14:33 ` Yu Ke
20 siblings, 0 replies; 22+ messages in thread
From: Yu Ke @ 2011-01-24 14:33 UTC (permalink / raw)
To: poky
Signed-off-by: Yu Ke <ke.yu@intel.com>
---
bitbake/lib/bb/fetch2/ssh.py | 2 ++
1 files changed, 2 insertions(+), 0 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/ssh.py b/bitbake/lib/bb/fetch2/ssh.py
index 78f55a6..711e668 100644
--- a/bitbake/lib/bb/fetch2/ssh.py
+++ b/bitbake/lib/bb/fetch2/ssh.py
@@ -112,6 +112,8 @@ class SSH(Fetch):
commands.mkarg(ldir)
)
+ bb.fetch2.check_network_access(d, cmd)
+
(exitstatus, output) = commands.getstatusoutput(cmd)
if exitstatus != 0:
print(output)
--
1.7.0.4
^ permalink raw reply related [flat|nested] 22+ messages in thread