[PATCH] stgit: allow spaces in filenames

[PATCH] stgit: allow spaces in filenames

[Bryan Larsen]

The current version of stgit does not allow whitespace in filenames.  This patch fixes that.  It also speeds up operations on large filesets considerably.

Signed-off-by: Bryan Larsen <bryan.larsen@gmail.com>
---

 stgit/git.py  |  158 +++++++++++++++++++++++++++++++--------------------------
 stgit/main.py |    8 +--
 2 files changed, 90 insertions(+), 76 deletions(-)

diff --git a/stgit/git.py b/stgit/git.py
--- a/stgit/git.py
+++ b/stgit/git.py
@@ -101,6 +101,24 @@ def __output(cmd):
         raise GitException, '%s failed' % cmd
     return string
 
+# __run: runs cmd using spawnvp.
+#
+#  The shell is avoided so it won't mess up our arguments.
+# If args is very large, the command is run multiple times;
+# args is split xargs style:  cmd is passed on each invocation.
+# Unlike xargs, returns immediately if any non-zero return code 
+# is received.
+#
+def __run(cmd, args=None):
+    args_l=cmd.split()
+    if args is None:
+	args = []
+    for i in range(0, len(args)+1, 1000):
+	r=os.spawnvp(os.P_WAIT, args_l[0], args_l + args[i:min(i+1000, len(args))])
+	if r:
+	    return r
+    return 0
+
 def __check_base_dir():
     return os.path.isdir(base_dir)
 
@@ -114,14 +132,16 @@ def __tree_status(files = [], tree_id = 
     # unknown files
     if unknown:
         exclude_file = os.path.join(base_dir, 'exclude')
-        extra_exclude = ''
+        extra_exclude = []
         if os.path.exists(exclude_file):
-            extra_exclude += ' --exclude-from=%s' % exclude_file
-        fout = os.popen('git-ls-files --others'
-                        ' --exclude="*.[ao]" --exclude=".*"'
-                        ' --exclude=TAGS --exclude=tags --exclude="*~"'
-                        ' --exclude="#*"' + extra_exclude, 'r')
+            extra_exclude.append('--exclude-from=%s' % exclude_file)
+        fin, fout = os.popen2(['git-ls-files', '--others',
+                        '--exclude=*.[ao]', '--exclude=.*'
+                        '--exclude=TAGS', '--exclude=tags', '--exclude=*~',
+                        '--exclude=#*'] + extra_exclude)
         cache_files += [('?', line.strip()) for line in fout]
+	fin.close()
+	fout.close()
 
     # conflicted files
     conflicts = get_conflicts()
@@ -130,13 +150,12 @@ def __tree_status(files = [], tree_id = 
     cache_files += [('C', filename) for filename in conflicts]
 
     # the rest
-    files_str = reduce(lambda x, y: x + ' ' + y, files, '')
-    fout = os.popen('git-diff-cache -r %s %s' % (tree_id, files_str), 'r')
+    fin, fout = os.popen2(['git-diff-cache', '-r', tree_id] + files)
     for line in fout:
-        fs = tuple(line.split()[4:])
+        fs = tuple(line.rstrip().split(' ',4)[-1].split('\t',1))
         if fs[1] not in conflicts:
             cache_files.append(fs)
-    if fout.close():
+    if fout.close() or fin.close():
         raise GitException, 'git-diff-cache failed'
 
     return cache_files
@@ -185,10 +204,9 @@ def add(names):
         else:
             raise GitException, '%s is not a file or directory' % i
 
-    for f in files:
-        print 'Adding file %s' % f
-        if os.system('git-update-cache --add -- %s' % f) != 0:
-            raise GitException, 'Unable to add %s' % f
+    if files:
+	if __run('git-update-cache --add --', files):
+            raise GitException, 'Unable to add file'
 
 def rm(files, force = False):
     """Remove a file from the repository
@@ -198,26 +216,32 @@ def rm(files, force = False):
     else:
         git_opt = '--remove'
 
-    for f in files:
-        if force:
-            print 'Removing file %s' % f
-            if os.system('git-update-cache --force-remove -- %s' % f) != 0:
-                raise GitException, 'Unable to remove %s' % f
-        elif os.path.exists(f):
-            raise GitException, '%s exists. Remove it first' %f
-        else:
-            print 'Removing file %s' % f
-            if os.system('git-update-cache --remove -- %s' % f) != 0:
-                raise GitException, 'Unable to remove %s' % f
+    if not force:
+	for f in files:
+	    if os.path.exists(f):
+		raise GitException, '%s exists. Remove it first' %f
+	if files:
+	    __run('git-update-cache --remove --', files)
+    else:
+	if files:
+	    __run('git-update-cache --force-remove --', files)
 
 def update_cache(files):
     """Update the cache information for the given files
     """
+    files_here = []
+    files_gone = []
+
     for f in files:
         if os.path.exists(f):
-            os.system('git-update-cache -- %s' % f)
+	    files_here.append(f)
         else:
-            os.system('git-update-cache --remove -- %s' % f)
+	    files_gone.append(f)
+
+    if files_here:
+	__run('git-update-cache --', files_here)
+    if files_gone:
+	__run('git-update-cache --remove --', files_gone)
 
 def commit(message, files = [], parents = [], allowempty = False,
            author_name = None, author_email = None, author_date = None,
@@ -248,16 +272,26 @@ def commit(message, files = [], parents 
 
     # update the cache
     if not first:
+	add_files=[]
+	rm_files=[]
+	m_files=[]
         for f in cache_files:
             if f[0] == 'N':
-                git_flag = '--add'
+		add_files.append(f[1])
             elif f[0] == 'D':
-                git_flag = '--force-remove'
+		rm_files.append(f[1])
             else:
-                git_flag = '--'
+                m_files.append(f[1])
 
-            if os.system('git-update-cache %s %s' % (git_flag, f[1])) != 0:
-                raise GitException, 'Failed git-update-cache -- %s' % f[1]
+	if add_files:
+	    if __run('git-update-cache --add --', add_files):
+		raise GitException, 'Failed git-update-cache --add'
+	if rm_files:
+	    if __run('git-update-cache --force-remove --', rm_files):
+		raise GitException, 'Failed git-update-cache --rm'
+	if m_files:
+	    if __run('git-update-cache --', m_files):
+		raise GitException, 'Failed git-update-cache'
 
     # write the index to repository
     tree_id = __output('git-write-tree')
@@ -292,7 +326,7 @@ def merge(base, head1, head2):
     """Perform a 3-way merge between base, head1 and head2 into the
     local tree
     """
-    if os.system('git-read-tree -u -m %s %s %s' % (base, head1, head2)) != 0:
+    if __run('git-read-tree -u -m', [base, head1, head2]) != 0:
         raise GitException, 'git-read-tree failed (local changes maybe?)'
 
     # this can fail if there are conflicts
@@ -330,51 +364,31 @@ def status(files = [], modified = False,
         else:
             print '%s' % fs[1]
 
-def diff(files = [], rev1 = 'HEAD', rev2 = None, output = None,
-         append = False):
+def diff(files = [], rev1 = 'HEAD', rev2 = None, out_fd = sys.stdout):
     """Show the diff between rev1 and rev2
     """
-    files_str = reduce(lambda x, y: x + ' ' + y, files, '')
-
-    extra_args = ''
-    if output:
-        if append:
-            extra_args += ' >> %s' % output
-        else:
-            extra_args += ' > %s' % output
-
     os.system('git-update-cache --refresh > /dev/null')
 
     if rev2:
-        if os.system('git-diff-tree -p %s %s %s %s'
-                     % (rev1, rev2, files_str, extra_args)) != 0:
-            raise GitException, 'git-diff-tree failed'
+	nf, in_fd = os.popen2(['git-diff-tree', '-p', rev1, rev2]+files)
     else:
-        if os.system('git-diff-cache -p %s %s %s'
-                     % (rev1, files_str, extra_args)) != 0:
-            raise GitException, 'git-diff-cache failed'
+	nf, in_fd = os.popen2(['git-diff-cache', '-p', rev1]+files)
+    
+    out_fd.write(in_fd.read())
+    if in_fd.close() or nf.close():
+	raise GitException, 'git.diff failed'
 
 def diffstat(files = [], rev1 = 'HEAD', rev2 = None):
     """Return the diffstat between rev1 and rev2
     """
-    files_str = reduce(lambda x, y: x + ' ' + y, files, '')
 
     os.system('git-update-cache --refresh > /dev/null')
-    ds_cmd = '| git-apply --stat'
-
-    if rev2:
-        f = os.popen('git-diff-tree -p %s %s %s %s'
-                     % (rev1, rev2, files_str, ds_cmd), 'r')
-        str = f.read().rstrip()
-        if f.close():
-            raise GitException, 'git-diff-tree failed'
-    else:
-        f = os.popen('git-diff-cache -p %s %s %s'
-                     % (rev1, files_str, ds_cmd), 'r')
-        str = f.read().rstrip()
-        if f.close():
-            raise GitException, 'git-diff-cache failed'
-
+    ds_in, ds_out = os.popen2('git-apply --stat')
+    diff(files, rev1, rev2, ds_in)
+    ds_in.close()
+    str = ds_out.read().rstrip()
+    if ds_out.close():
+	raise GitException, 'git.diffstat failed'
     return str
 
 def files(rev1, rev2):
@@ -386,7 +400,7 @@ def files(rev1, rev2):
     f = os.popen('git-diff-tree -r %s %s' % (rev1, rev2),
                  'r')
     for line in f:
-        str += '%s %s\n' % tuple(line.split()[4:])
+        str += '%s %s\n' % tuple(line.rstrip().split(' ',4)[-1].split('\t',1))
     if f.close():
         raise GitException, 'git-diff-tree failed'
 
@@ -395,23 +409,23 @@ def files(rev1, rev2):
 def checkout(files = [], force = False):
     """Check out the given or all files
     """
-    git_flags = ''
+    git_flags = 'git-checkout-cache -q -u'
     if force:
         git_flags += ' -f'
     if len(files) == 0:
         git_flags += ' -a'
     else:
-        git_flags += reduce(lambda x, y: x + ' ' + y, files, ' --')
+	git_flags += ' --'
 
-    if os.system('git-checkout-cache -q -u%s' % git_flags) != 0:
-        raise GitException, 'Failed git-checkout-cache -q -u%s' % git_flags
+    if __run(git_flags, files) != 0:
+        raise GitException, 'Failed git-checkout-cache'
 
 def switch(tree_id):
     """Switch the tree to the given id
     """
     to_delete = filter(lambda x: x[0] == 'N', __tree_status(tree_id = tree_id))
 
-    if os.system('git-read-tree -m %s' % tree_id) != 0:
+    if __run('git-read-tree -m', [tree_id]) != 0:
         raise GitException, 'Failed git-read-tree -m %s' % tree_id
 
     checkout(force = True)
diff --git a/stgit/main.py b/stgit/main.py
--- a/stgit/main.py
+++ b/stgit/main.py
@@ -720,13 +720,13 @@ def export(parser, options, args):
                   'supported in the patch template'
         f = open(pfile, 'w+')
         f.write(descr)
-        f.close()
 
         # write the diff
-        git.diff(rev1 = __git_id('%s/bottom' % p),
+	git.diff(rev1 = __git_id('%s/bottom' % p),
                  rev2 = __git_id('%s/top' % p),
-                 output = pfile, append = True)
-        patch_no += 1
+                 out_fd = f)
+	f.close()
+	patch_no += 1
 
     series.close()
 

[PATCH] stgit: new --message does not work

[Bryan Larsen]

Stacked GIT: Using the --message option with the new command fails.  Fix it.

Signed-off-by: Bryan Larsen <bryan.larsen@gmail.com>
---

 stgit/stack.py |    2 ++
 1 files changed, 2 insertions(+), 0 deletions(-)

diff --git a/stgit/stack.py b/stgit/stack.py
--- a/stgit/stack.py
+++ b/stgit/stack.py
@@ -384,6 +384,8 @@ class Series:
             descr = edit_file(None, \
                               'Please enter the description for patch "%s" ' \
                               'above.' % name)
+	else:
+	    descr = message
 
         head = git.get_head()
 

Re: [PATCH] stgit: allow spaces in filenames

[Catalin Marinas]

Bryan Larsen <bryanlarsen@yahoo.com> wrote:
> The current version of stgit does not allow whitespace in filenames.
> This patch fixes that.  It also speeds up operations on large
> filesets considerably.

Thanks, I will apply it but I have a few comments below:

> +# __run: runs cmd using spawnvp.
> +#
> +#  The shell is avoided so it won't mess up our arguments.
> +# If args is very large, the command is run multiple times;
> +# args is split xargs style:  cmd is passed on each invocation.
> +# Unlike xargs, returns immediately if any non-zero return code 
> +# is received.
> +#
> +def __run(cmd, args=None):

I would prefer to add this as Python function documentation, i.e. with
"""...""" in the function body.

An additional thing, can you please convert all the tabs to spaces?
That's a better convention for a language like Python where you
delimit blocks by indentation.

> @@ -114,14 +132,16 @@ def __tree_status(files = [], tree_id = 
>      # unknown files
>      if unknown:
>          exclude_file = os.path.join(base_dir, 'exclude')
> -        extra_exclude = ''
> +        extra_exclude = []
>          if os.path.exists(exclude_file):
> -            extra_exclude += ' --exclude-from=%s' % exclude_file
> -        fout = os.popen('git-ls-files --others'
> -                        ' --exclude="*.[ao]" --exclude=".*"'
> -                        ' --exclude=TAGS --exclude=tags --exclude="*~"'
> -                        ' --exclude="#*"' + extra_exclude, 'r')
> +            extra_exclude.append('--exclude-from=%s' % exclude_file)
> +        fin, fout = os.popen2(['git-ls-files', '--others',
> +                        '--exclude=*.[ao]', '--exclude=.*'
> +                        '--exclude=TAGS', '--exclude=tags', '--exclude=*~',
> +                        '--exclude=#*'] + extra_exclude)
>          cache_files += [('?', line.strip()) for line in fout]
> +	fin.close()
> +	fout.close()

What's the reason for having 'fin' as well? It doesn't seem to be used
(this is found in other parts of the patch as well).

Something wrong happened to my git-ftp-push script. It looks like it
copied 'master' to stgit.git/ and not to stgit.git/refs/heads/. I
can't fix it until tonight. Before then, you could actually use wget
to pull the whole repository and just copy 'master' to
'refs/heads/master'. In the latest tree I created separate files for
each command.

I'm not sure whether the GIT guys are happy for us to use this mailing
list for StGIT. If the StGIT traffic increases, I will try to create a
separate mailing list (maybe using a site like sf.net).

-- 
Catalin

Re: [PATCH] stgit: allow spaces in filenames

[Bryan Larsen]

Catalin Marinas wrote:
> An additional thing, can you please convert all the tabs to spaces?
> That's a better convention for a language like Python where you
> delimit blocks by indentation.

I would have hoped that emacs py-mode would "do the right thing". 
Anybody know how to make it do what Catalin wants?

> 
> 
>>+        fin, fout = os.popen2(['git-ls-files', '--others',
> 
> 
> What's the reason for having 'fin' as well? It doesn't seem to be used
> (this is found in other parts of the patch as well).

popen does not support bypassing the shell by using vectors of 
arguments.  Only popen2 and friends have this capability.

Unfortunate, yes.

> 
> I'm not sure whether the GIT guys are happy for us to use this mailing
> list for StGIT. If the StGIT traffic increases, I will try to create a
> separate mailing list (maybe using a site like sf.net).
> 

I'd very much like to stay on the same list.  By the same logic, cogito 
should have it's own list as well...

Bryan

Re: [PATCH] stgit: allow spaces in filenames

[Jerry Seutter]

Bryan Larsen wrote:
> Catalin Marinas wrote:
> 
>> An additional thing, can you please convert all the tabs to spaces?
>> That's a better convention for a language like Python where you
>> delimit blocks by indentation.
> 
> 
> I would have hoped that emacs py-mode would "do the right thing". 
> Anybody know how to make it do what Catalin wants?

Yeah, the default emacs mode seems to be to Do The Wrong Thing.  I have 
this in my .emacs, YMMV.

(setq-default indent-tabs-mode nil)     ; Don't insert tab characters.
(setq-default tab-width 4)              ; If there are tabs, display
                                         ; as 4 spaces.

You can set the tab-width to something much larger to make existing tabs 
obvious.

Jerry

Re: [PATCH] stgit: allow spaces in filenames

[Catalin Marinas]

On Wed, 2005-07-13 at 14:17 -0400, Bryan Larsen wrote:
> Catalin Marinas wrote:
> I would have hoped that emacs py-mode would "do the right thing". 
> Anybody know how to make it do what Catalin wants?

It looks like the python-mode in my emacs does the right thing. You
could add something like below in your .emacs file:

(add-hook 'python-mode-hook
          #'(lambda ()
              (setq indent-tabs-mode nil)))

Otherwise, select the whole buffer and do a "M-x untabify".

> > What's the reason for having 'fin' as well? It doesn't seem to be used
> > (this is found in other parts of the patch as well).
> 
> popen does not support bypassing the shell by using vectors of 
> arguments.  Only popen2 and friends have this capability.

But the manual says that it is not possible to get the exit code of the
child process with popen2 (at least not in python 2.3). You would need
to use the Popen3 and Popen4 classes in the popen2 module.

Does it make that big difference if the commands are invoked via the
shell? I haven't run any tests.

> > I'm not sure whether the GIT guys are happy for us to use this mailing
> > list for StGIT. If the StGIT traffic increases, I will try to create a
> > separate mailing list (maybe using a site like sf.net).
> > 
> 
> I'd very much like to stay on the same list.  By the same logic, cogito 
> should have it's own list as well...

I'd like this too and it's probably OK with a low traffic (we'll see if
we receive complaints :-) ).

Catalin

Re: [PATCH] stgit: allow spaces in filenames

[Junio C Hamano]

Catalin Marinas <catalin.marinas@gmail.com> writes:

>> I'd very much like to stay on the same list.  By the same logic, cogito 
>> should have it's own list as well...
>
> I'd like this too and it's probably OK with a low traffic (we'll see if
> we receive complaints :-) ).

I'd like to keep Porcelain discussions on this list for two
reasons:

 (1) Porcelain implementations need their own bookkeeping
     information somewhere, and possibly in .git/ directory.
     I'd want to see them agree on what's stored where for what
     purpose and stay compatible when that makes sense.  The
     place to brew that concensus is here.

 (2) The core GIT people want to learn what Porcelain needs
     from the core.  I am personally interested in StGIT so
     subscribing to that list is fine for me, but it is
     convenient to have everything in one place, especially with
     this relatively low traffic.

Re: [PATCH] stgit: allow spaces in filenames

[Catalin Marinas]

On Wed, 2005-07-13 at 15:26 -0700, Junio C Hamano wrote:
> Catalin Marinas <catalin.marinas@gmail.com> writes:
> 
> >> I'd very much like to stay on the same list.  By the same logic, cogito 
> >> should have it's own list as well...
> >
> > I'd like this too and it's probably OK with a low traffic (we'll see if
> > we receive complaints :-) ).
> 
> I'd like to keep Porcelain discussions on this list for two
> reasons:

That's great then. I will keep the StGIT discussions on this list.

--
Catalin

Re: [PATCH] stgit: allow spaces in filenames

[Bryan Larsen]

> Does it make that big difference if the commands are invoked via the
> shell? I haven't run any tests.
> 

It wasn't for the time difference that I bypassed the shell, it was to 
support spaces and other strange characters  in parameters.  It's easy 
to use spawnvp than it is to escape the parameters properly.

Bryan

Re: [PATCH] stgit: allow spaces in filenames

[Matthias Urlichs]

Hi, Bryan Larsen wrote:

> +	r=os.spawnvp(os.P_WAIT, args_l[0], args_l + args[i:min(i+1000, len(args))])

The max length for argv is 32k IIRC, so 1000 is 28-byte file names.
That's probably not enough. I't suggest using chunks of 100.

-- 
Matthias Urlichs   |   {M:U} IT Design @ m-u-it.de   |  smurf@smurf.noris.de
Disclaimer: The quote was selected randomly. Really. | http://smurf.noris.de
 - -
Don't you feel more like you do now than you did when you came in?

Re: [PATCH] stgit: allow spaces in filenames

[Linus Torvalds]

On Wed, 13 Jul 2005, Matthias Urlichs wrote:
>
> Hi, Bryan Larsen wrote:
> 
> > +	r=os.spawnvp(os.P_WAIT, args_l[0], args_l + args[i:min(i+1000, len(args))])
> 
> The max length for argv is 32k IIRC, so 1000 is 28-byte file names.

I think 32k may be the posix-mandated minimum. Linux does 128kB, so 1000 
should likely be ok for most projects.

That said, xargs should do things like that right.

		Linus