Re: [PATCH 1/1] bb/fetch2: fix the "filename too long" error

[Richard Purdie <richard.purdie@linuxfoundation.org>]

On Tue, 2013-10-29 at 22:24 +0800, Robert Yang wrote:
> When the SRC_URI is very long or the PREMIRROR/MIRROR is in a very deep
> directory, the git, svn and other fetchers will covert the path into
> the filename, so if:
> 
>   len(path) > NAME_MAX,
> 
> then we will get the "filename too long" error, the oe-core supports to
> build when len(TMPDIR) <= 410, but the NAME_MAX is usually 256, so we
> can easily get this error when we put the PREMIRROR/MIRROR in a deep
> dir.
> 
> Truncate the filename to NAME_MAX - 10 will fix the problem, the "-10"
> is used for the ".done" and ".lock", in fact, -5 would be OK, but use
> "-10" in case we will have other longer suffix in the future.
> 
> [YOCTO #5389]

Aren't the "long SRC_URI" and deep PREMIRROR/MIRROR problems different
issues?

As far as I know, we don't have any long SRC_URIs in use anywhere within
the project. We could just error if we found one with a path we couldn't
cope with.

The deep PREMIRROR/MIRROR issue on the other hand sounds like something
we should adjust the code to handle properly?

Randomly truncating path names like this looks like a rather hacky
solution and it loses key information, there is a reason these long
names are being used as keys.

Put another way, the mirror tarball name should not change depending on
how deep the DL_DIR or other directories are.

So no, there is no way this patch is correct or should be merged, sorry.

Cheers,

Richard



> Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
> ---
>  bitbake/lib/bb/fetch2/__init__.py | 18 ++++++++++++++++++
>  bitbake/lib/bb/fetch2/bzr.py      |  2 ++
>  bitbake/lib/bb/fetch2/git.py      |  6 ++++++
>  bitbake/lib/bb/fetch2/hg.py       |  3 +++
>  bitbake/lib/bb/fetch2/osc.py      |  2 ++
>  bitbake/lib/bb/fetch2/repo.py     |  4 ++++
>  bitbake/lib/bb/fetch2/svk.py      |  2 ++
>  bitbake/lib/bb/fetch2/svn.py      |  3 +++
>  8 files changed, 40 insertions(+)
> 
> diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
> index 451d104..f76568e 100644
> --- a/bitbake/lib/bb/fetch2/__init__.py
> +++ b/bitbake/lib/bb/fetch2/__init__.py
> @@ -959,6 +959,24 @@ def get_file_checksums(filelist, pn):
>      checksums.sort(key=operator.itemgetter(1))
>      return checksums
>  
> +def truncate_to_name_max(dirpath, filename):
> +    """
> +    Truncate the filename to meet len(filename) <= NAME_MAX -10, return
> +    the last NAME_MAX - 10 characters if it doesn't, the -10 is for the
> +    '.done', '.lock' and perhaps others in the future.
> +    """
> +    if not os.path.exists(dirpath):
> +        bb.utils.mkdirhier(dirpath)
> +
> +    name_max = os.pathconf(dirpath, 'PC_NAME_MAX')
> +    new_max = name_max - 10
> +    if len(filename) > new_max:
> +        new_name = filename[-new_max:]
> +        logger.debug(1, "Truncate %s to %s to meet %s (NAME_MAX - 10) characters" % \
> +                    (filename, new_name, new_max))
> +        filename = new_name
> +
> +    return filename
>  
>  class FetchData(object):
>      """
> diff --git a/bitbake/lib/bb/fetch2/bzr.py b/bitbake/lib/bb/fetch2/bzr.py
> index 5d9e5f9..f91c7ee 100644
> --- a/bitbake/lib/bb/fetch2/bzr.py
> +++ b/bitbake/lib/bb/fetch2/bzr.py
> @@ -51,6 +51,8 @@ class Bzr(FetchMethod):
>              ud.revision = self.latest_revision(ud.url, ud, d)
>  
>          ud.localfile = data.expand('bzr_%s_%s_%s.tar.gz' % (ud.host, ud.path.replace('/', '.'), ud.revision), d)
> +        # The length of the filename should be less than NAME_MAX - 10
> +        ud.localfile = bb.fetch2.truncate_to_name_max(ud.pkgdir, ud.localfile)
>  
>      def _buildbzrcommand(self, ud, d, command):
>          """
> diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
> index 6175e4c..19598ef 100644
> --- a/bitbake/lib/bb/fetch2/git.py
> +++ b/bitbake/lib/bb/fetch2/git.py
> @@ -135,9 +135,15 @@ class Git(FetchMethod):
>          if ud.rebaseable:
>              for name in ud.names:
>                  gitsrcname = gitsrcname + '_' + ud.revisions[name]
> +
>          ud.mirrortarball = 'git2_%s.tar.gz' % (gitsrcname)
> +        # The length of the filename should be less than NAME_MAX - 10
> +        ud.mirrortarball = bb.fetch2.truncate_to_name_max(d.getVar("DL_DIR", True), ud.mirrortarball)
>          ud.fullmirror = os.path.join(d.getVar("DL_DIR", True), ud.mirrortarball)
> +
>          gitdir = d.getVar("GITDIR", True) or (d.getVar("DL_DIR", True) + "/git2/")
> +        # The length of the filename should be less than NAME_MAX - 10
> +        gitsrcname = bb.fetch2.truncate_to_name_max(gitdir, gitsrcname)
>          ud.clonedir = os.path.join(gitdir, gitsrcname)
>  
>          ud.localfile = ud.clonedir
> diff --git a/bitbake/lib/bb/fetch2/hg.py b/bitbake/lib/bb/fetch2/hg.py
> index b1c8675..be9bb26 100644
> --- a/bitbake/lib/bb/fetch2/hg.py
> +++ b/bitbake/lib/bb/fetch2/hg.py
> @@ -66,6 +66,9 @@ class Hg(FetchMethod):
>  
>          ud.localfile = data.expand('%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision), d)
>  
> +        # The length of the filename should be less than NAME_MAX - 10
> +        ud.localfile = bb.fetch2.truncate_to_name_max(ud.pkgdir, ud.localfile)
> +
>      def need_update(self, url, ud, d):
>          revTag = ud.parm.get('rev', 'tip')
>          if revTag == "tip":
> diff --git a/bitbake/lib/bb/fetch2/osc.py b/bitbake/lib/bb/fetch2/osc.py
> index 1a3a7bb..2e276e0 100644
> --- a/bitbake/lib/bb/fetch2/osc.py
> +++ b/bitbake/lib/bb/fetch2/osc.py
> @@ -48,6 +48,8 @@ class Osc(FetchMethod):
>                  ud.revision = ""
>  
>          ud.localfile = data.expand('%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.path.replace('/', '.'), ud.revision), d)
> +        # The length of the filename should be less than NAME_MAX - 10
> +        ud.localfile = bb.fetch2.truncate_to_name_max(ud.pkgdir, ud.localfile)
>  
>      def _buildosccommand(self, ud, d, command):
>          """
> diff --git a/bitbake/lib/bb/fetch2/repo.py b/bitbake/lib/bb/fetch2/repo.py
> index 8300da8..1e46968 100644
> --- a/bitbake/lib/bb/fetch2/repo.py
> +++ b/bitbake/lib/bb/fetch2/repo.py
> @@ -52,6 +52,8 @@ class Repo(FetchMethod):
>              ud.manifest += '.xml'
>  
>          ud.localfile = data.expand("repo_%s%s_%s_%s.tar.gz" % (ud.host, ud.path.replace("/", "."), ud.manifest, ud.branch), d)
> +        # The length of the filename should be less than NAME_MAX - 10
> +        ud.localfile = bb.fetch2.truncate_to_name_max(d.getVar("DL_DIR", True), ud.localfile)
>  
>      def download(self, loc, ud, d):
>          """Fetch url"""
> @@ -62,6 +64,8 @@ class Repo(FetchMethod):
>  
>          gitsrcname = "%s%s" % (ud.host, ud.path.replace("/", "."))
>          repodir = data.getVar("REPODIR", d, True) or os.path.join(data.getVar("DL_DIR", d, True), "repo")
> +        # The length of the filename should be less than NAME_MAX - 10
> +        gitsrcname = bb.fetch2.truncate_to_name_max(repodir, gitsrcname)
>          codir = os.path.join(repodir, gitsrcname, ud.manifest)
>  
>          if ud.user:
> diff --git a/bitbake/lib/bb/fetch2/svk.py b/bitbake/lib/bb/fetch2/svk.py
> index ee3823f..dbd78f3 100644
> --- a/bitbake/lib/bb/fetch2/svk.py
> +++ b/bitbake/lib/bb/fetch2/svk.py
> @@ -53,6 +53,8 @@ class Svk(FetchMethod):
>          ud.revision = ud.parm.get('rev', "")
>  
>          ud.localfile = data.expand('%s_%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision, ud.date), d)
> +        # The length of the filename should be less than NAME_MAX - 10
> +        ud.localfile = bb.fetch2.truncate_to_name_max(d.getVar("DL_DIR", True), ud.localfile)
>  
>      def need_update(self, url, ud, d):
>          if ud.date == "now":
> diff --git a/bitbake/lib/bb/fetch2/svn.py b/bitbake/lib/bb/fetch2/svn.py
> index 9a779d2..c155735 100644
> --- a/bitbake/lib/bb/fetch2/svn.py
> +++ b/bitbake/lib/bb/fetch2/svn.py
> @@ -65,6 +65,9 @@ class Svn(FetchMethod):
>  
>          ud.localfile = data.expand('%s_%s_%s_%s_.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision), d)
>  
> +        # The length of the filename should be less than NAME_MAX - 10
> +        ud.localfile = bb.fetch2.truncate_to_name_max(ud.pkgdir, ud.localfile)
> +
>      def _buildsvncommand(self, ud, d, command):
>          """
>          Build up an svn commandline based on ud