threads / patch / 1219

patchstgit: allow spaces in filenames

Subject: [PATCH] stgit: allow spaces in filenames

## tl;dr

11 messages between Jul 13, 2005 and Jul 14, 2005. Diffs are folded; open one to read it.

replies: 10people: 6as markdown or json

Bryan Larsen· Jul 13, 2005, 08:38 UTC · lore
The current version of stgit does not allow whitespace in filenames.  This patch fixes that.  It also speeds up operations on large filesets considerably.
Signed-off-by: Bryan Larsen <bryan.larsen@gmail.com>
---
 stgit/git.py  |  158 +++++++++++++++++++++++++++++++--------------------------
 stgit/main.py |    8 +--
 2 files changed, 90 insertions(+), 76 deletions(-)
Show changes to 2 files +90 −76

stgit/git.py, stgit/main.py

diff --git a/stgit/git.py b/stgit/git.py
--- a/stgit/git.py
+++ b/stgit/git.py
@@ -101,6 +101,24 @@ def __output(cmd):
         raise GitException, '%s failed' % cmd
     return string
 
+# __run: runs cmd using spawnvp.
+#
+#  The shell is avoided so it won't mess up our arguments.
+# If args is very large, the command is run multiple times;
+# args is split xargs style:  cmd is passed on each invocation.
+# Unlike xargs, returns immediately if any non-zero return code 
+# is received.
+#
+def __run(cmd, args=None):
+    args_l=cmd.split()
+    if args is None:
+	args = []
+    for i in range(0, len(args)+1, 1000):
+	r=os.spawnvp(os.P_WAIT, args_l[0], args_l + args[i:min(i+1000, len(args))])
+	if r:
+	    return r
+    return 0
+
 def __check_base_dir():
     return os.path.isdir(base_dir)
 
@@ -114,14 +132,16 @@ def __tree_status(files = [], tree_id = 
     # unknown files
     if unknown:
         exclude_file = os.path.join(base_dir, 'exclude')
-        extra_exclude = ''
+        extra_exclude = []
         if os.path.exists(exclude_file):
-            extra_exclude += ' --exclude-from=%s' % exclude_file
-        fout = os.popen('git-ls-files --others'
-                        ' --exclude="*.[ao]" --exclude=".*"'
-                        ' --exclude=TAGS --exclude=tags --exclude="*~"'
-                        ' --exclude="#*"' + extra_exclude, 'r')
+            extra_exclude.append('--exclude-from=%s' % exclude_file)
+        fin, fout = os.popen2(['git-ls-files', '--others',
+                        '--exclude=*.[ao]', '--exclude=.*'
+                        '--exclude=TAGS', '--exclude=tags', '--exclude=*~',
+                        '--exclude=#*'] + extra_exclude)
         cache_files += [('?', line.strip()) for line in fout]
+	fin.close()
+	fout.close()
 
     # conflicted files
     conflicts = get_conflicts()
@@ -130,13 +150,12 @@ def __tree_status(files = [], tree_id = 
     cache_files += [('C', filename) for filename in conflicts]
 
     # the rest
-    files_str = reduce(lambda x, y: x + ' ' + y, files, '')
-    fout = os.popen('git-diff-cache -r %s %s' % (tree_id, files_str), 'r')
+    fin, fout = os.popen2(['git-diff-cache', '-r', tree_id] + files)
     for line in fout:
-        fs = tuple(line.split()[4:])
+        fs = tuple(line.rstrip().split(' ',4)[-1].split('\t',1))
         if fs[1] not in conflicts:
             cache_files.append(fs)
-    if fout.close():
+    if fout.close() or fin.close():
         raise GitException, 'git-diff-cache failed'
 
     return cache_files
@@ -185,10 +204,9 @@ def add(names):
         else:
             raise GitException, '%s is not a file or directory' % i
 
-    for f in files:
-        print 'Adding file %s' % f
-        if os.system('git-update-cache --add -- %s' % f) != 0:
-            raise GitException, 'Unable to add %s' % f
+    if files:
+	if __run('git-update-cache --add --', files):
+            raise GitException, 'Unable to add file'
 
 def rm(files, force = False):
     """Remove a file from the repository
@@ -198,26 +216,32 @@ def rm(files, force = False):
     else:
         git_opt = '--remove'
 
-    for f in files:
-        if force:
-            print 'Removing file %s' % f
-            if os.system('git-update-cache --force-remove -- %s' % f) != 0:
-                raise GitException, 'Unable to remove %s' % f
-        elif os.path.exists(f):
-            raise GitException, '%s exists. Remove it first' %f
-        else:
-            print 'Removing file %s' % f
-            if os.system('git-update-cache --remove -- %s' % f) != 0:
-                raise GitException, 'Unable to remove %s' % f
+    if not force:
+	for f in files:
+	    if os.path.exists(f):
+		raise GitException, '%s exists. Remove it first' %f
+	if files:
+	    __run('git-update-cache --remove --', files)
+    else:
+	if files:
+	    __run('git-update-cache --force-remove --', files)
 
 def update_cache(files):
     """Update the cache information for the given files
     """
+    files_here = []
+    files_gone = []
+
     for f in files:
         if os.path.exists(f):
-            os.system('git-update-cache -- %s' % f)
+	    files_here.append(f)
         else:
-            os.system('git-update-cache --remove -- %s' % f)
+	    files_gone.append(f)
+
+    if files_here:
+	__run('git-update-cache --', files_here)
+    if files_gone:
+	__run('git-update-cache --remove --', files_gone)
 
 def commit(message, files = [], parents = [], allowempty = False,
            author_name = None, author_email = None, author_date = None,
@@ -248,16 +272,26 @@ def commit(message, files = [], parents 
 
     # update the cache
     if not first:
+	add_files=[]
+	rm_files=[]
+	m_files=[]
         for f in cache_files:
             if f[0] == 'N':
-                git_flag = '--add'
+		add_files.append(f[1])
             elif f[0] == 'D':
-                git_flag = '--force-remove'
+		rm_files.append(f[1])
             else:
-                git_flag = '--'
+                m_files.append(f[1])
 
-            if os.system('git-update-cache %s %s' % (git_flag, f[1])) != 0:
-                raise GitException, 'Failed git-update-cache -- %s' % f[1]
+	if add_files:
+	    if __run('git-update-cache --add --', add_files):
+		raise GitException, 'Failed git-update-cache --add'
+	if rm_files:
+	    if __run('git-update-cache --force-remove --', rm_files):
+		raise GitException, 'Failed git-update-cache --rm'
+	if m_files:
+	    if __run('git-update-cache --', m_files):
+		raise GitException, 'Failed git-update-cache'
 
     # write the index to repository
     tree_id = __output('git-write-tree')
@@ -292,7 +326,7 @@ def merge(base, head1, head2):
     """Perform a 3-way merge between base, head1 and head2 into the
     local tree
     """
-    if os.system('git-read-tree -u -m %s %s %s' % (base, head1, head2)) != 0:
+    if __run('git-read-tree -u -m', [base, head1, head2]) != 0:
         raise GitException, 'git-read-tree failed (local changes maybe?)'
 
     # this can fail if there are conflicts
@@ -330,51 +364,31 @@ def status(files = [], modified = False,
         else:
             print '%s' % fs[1]
 
-def diff(files = [], rev1 = 'HEAD', rev2 = None, output = None,
-         append = False):
+def diff(files = [], rev1 = 'HEAD', rev2 = None, out_fd = sys.stdout):
     """Show the diff between rev1 and rev2
     """
-    files_str = reduce(lambda x, y: x + ' ' + y, files, '')
-
-    extra_args = ''
-    if output:
-        if append:
-            extra_args += ' >> %s' % output
-        else:
-            extra_args += ' > %s' % output
-
     os.system('git-update-cache --refresh > /dev/null')
 
     if rev2:
-        if os.system('git-diff-tree -p %s %s %s %s'
-                     % (rev1, rev2, files_str, extra_args)) != 0:
-            raise GitException, 'git-diff-tree failed'
+	nf, in_fd = os.popen2(['git-diff-tree', '-p', rev1, rev2]+files)
     else:
-        if os.system('git-diff-cache -p %s %s %s'
-                     % (rev1, files_str, extra_args)) != 0:
-            raise GitException, 'git-diff-cache failed'
+	nf, in_fd = os.popen2(['git-diff-cache', '-p', rev1]+files)
+    
+    out_fd.write(in_fd.read())
+    if in_fd.close() or nf.close():
+	raise GitException, 'git.diff failed'
 
 def diffstat(files = [], rev1 = 'HEAD', rev2 = None):
     """Return the diffstat between rev1 and rev2
     """
-    files_str = reduce(lambda x, y: x + ' ' + y, files, '')
 
     os.system('git-update-cache --refresh > /dev/null')
-    ds_cmd = '| git-apply --stat'
-
-    if rev2:
-        f = os.popen('git-diff-tree -p %s %s %s %s'
-                     % (rev1, rev2, files_str, ds_cmd), 'r')
-        str = f.read().rstrip()
-        if f.close():
-            raise GitException, 'git-diff-tree failed'
-    else:
-        f = os.popen('git-diff-cache -p %s %s %s'
-                     % (rev1, files_str, ds_cmd), 'r')
-        str = f.read().rstrip()
-        if f.close():
-            raise GitException, 'git-diff-cache failed'
-
+    ds_in, ds_out = os.popen2('git-apply --stat')
+    diff(files, rev1, rev2, ds_in)
+    ds_in.close()
+    str = ds_out.read().rstrip()
+    if ds_out.close():
+	raise GitException, 'git.diffstat failed'
     return str
 
 def files(rev1, rev2):
@@ -386,7 +400,7 @@ def files(rev1, rev2):
     f = os.popen('git-diff-tree -r %s %s' % (rev1, rev2),
                  'r')
     for line in f:
-        str += '%s %s\n' % tuple(line.split()[4:])
+        str += '%s %s\n' % tuple(line.rstrip().split(' ',4)[-1].split('\t',1))
     if f.close():
         raise GitException, 'git-diff-tree failed'
 
@@ -395,23 +409,23 @@ def files(rev1, rev2):
 def checkout(files = [], force = False):
     """Check out the given or all files
     """
-    git_flags = ''
+    git_flags = 'git-checkout-cache -q -u'
     if force:
         git_flags += ' -f'
     if len(files) == 0:
         git_flags += ' -a'
     else:
-        git_flags += reduce(lambda x, y: x + ' ' + y, files, ' --')
+	git_flags += ' --'
 
-    if os.system('git-checkout-cache -q -u%s' % git_flags) != 0:
-        raise GitException, 'Failed git-checkout-cache -q -u%s' % git_flags
+    if __run(git_flags, files) != 0:
+        raise GitException, 'Failed git-checkout-cache'
 
 def switch(tree_id):
     """Switch the tree to the given id
     """
     to_delete = filter(lambda x: x[0] == 'N', __tree_status(tree_id = tree_id))
 
-    if os.system('git-read-tree -m %s' % tree_id) != 0:
+    if __run('git-read-tree -m', [tree_id]) != 0:
         raise GitException, 'Failed git-read-tree -m %s' % tree_id
 
     checkout(force = True)
diff --git a/stgit/main.py b/stgit/main.py
--- a/stgit/main.py
+++ b/stgit/main.py
@@ -720,13 +720,13 @@ def export(parser, options, args):
                   'supported in the patch template'
         f = open(pfile, 'w+')
         f.write(descr)
-        f.close()
 
         # write the diff
-        git.diff(rev1 = __git_id('%s/bottom' % p),
+	git.diff(rev1 = __git_id('%s/bottom' % p),
                  rev2 = __git_id('%s/top' % p),
-                 output = pfile, append = True)
-        patch_no += 1
+                 out_fd = f)
+	f.close()
+	patch_no += 1
 
     series.close()
Bryan Larsen· Jul 13, 2005, 08:38 UTC · re: Bryan Larsen · lore

[PATCH] stgit: new --message does not work

Stacked GIT: Using the --message option with the new command fails.  Fix it.
Signed-off-by: Bryan Larsen <bryan.larsen@gmail.com>
---
 stgit/stack.py |    2 ++
 1 files changed, 2 insertions(+), 0 deletions(-)
Show changes to stgit/stack.py +2 −0
diff --git a/stgit/stack.py b/stgit/stack.py
--- a/stgit/stack.py
+++ b/stgit/stack.py
@@ -384,6 +384,8 @@ class Series:
             descr = edit_file(None, \
                               'Please enter the description for patch "%s" ' \
                               'above.' % name)
+	else:
+	    descr = message
 
         head = git.get_head()
Catalin Marinas· Jul 13, 2005, 10:17 UTC · re: Bryan Larsen · lore

Re: [PATCH] stgit: allow spaces in filenames

Bryan Larsen <bryanlarsen@yahoo.com> wrote:
> The current version of stgit does not allow whitespace in filenames.
> This patch fixes that.  It also speeds up operations on large
> filesets considerably.
Thanks, I will apply it but I have a few comments below:
Show 9 quoted lines
> +# __run: runs cmd using spawnvp.
> +#
> +#  The shell is avoided so it won't mess up our arguments.
> +# If args is very large, the command is run multiple times;
> +# args is split xargs style:  cmd is passed on each invocation.
> +# Unlike xargs, returns immediately if any non-zero return code 
> +# is received.
> +#
> +def __run(cmd, args=None):

I would prefer to add this as Python function documentation, i.e. with """...""" in the function body.

An additional thing, can you please convert all the tabs to spaces? That's a better convention for a language like Python where you delimit blocks by indentation.

Show 20 quoted lines
> @@ -114,14 +132,16 @@ def __tree_status(files = [], tree_id = 
>      # unknown files
>      if unknown:
>          exclude_file = os.path.join(base_dir, 'exclude')
> -        extra_exclude = ''
> +        extra_exclude = []
>          if os.path.exists(exclude_file):
> -            extra_exclude += ' --exclude-from=%s' % exclude_file
> -        fout = os.popen('git-ls-files --others'
> -                        ' --exclude="*.[ao]" --exclude=".*"'
> -                        ' --exclude=TAGS --exclude=tags --exclude="*~"'
> -                        ' --exclude="#*"' + extra_exclude, 'r')
> +            extra_exclude.append('--exclude-from=%s' % exclude_file)
> +        fin, fout = os.popen2(['git-ls-files', '--others',
> +                        '--exclude=*.[ao]', '--exclude=.*'
> +                        '--exclude=TAGS', '--exclude=tags', '--exclude=*~',
> +                        '--exclude=#*'] + extra_exclude)
>          cache_files += [('?', line.strip()) for line in fout]
> +	fin.close()
> +	fout.close()

What's the reason for having 'fin' as well? It doesn't seem to be used (this is found in other parts of the patch as well).

Something wrong happened to my git-ftp-push script. It looks like it copied 'master' to stgit.git/ and not to stgit.git/refs/heads/. I can't fix it until tonight. Before then, you could actually use wget to pull the whole repository and just copy 'master' to 'refs/heads/master'. In the latest tree I created separate files for each command.

I'm not sure whether the GIT guys are happy for us to use this mailing list for StGIT. If the StGIT traffic increases, I will try to create a separate mailing list (maybe using a site like sf.net).

-- 
Catalin
Bryan Larsen· Jul 13, 2005, 18:17 UTC · re: Catalin Marinas · lore

Re: [PATCH] stgit: allow spaces in filenames

Catalin Marinas wrote:
> An additional thing, can you please convert all the tabs to spaces?
> That's a better convention for a language like Python where you
> delimit blocks by indentation.

I would have hoped that emacs py-mode would "do the right thing". Anybody know how to make it do what Catalin wants?

Show 7 quoted lines
> 
> 
>>+        fin, fout = os.popen2(['git-ls-files', '--others',
> 
> 
> What's the reason for having 'fin' as well? It doesn't seem to be used
> (this is found in other parts of the patch as well).

popen does not support bypassing the shell by using vectors of arguments. Only popen2 and friends have this capability.

Unfortunate, yes.
Show 5 quoted lines
> 
> I'm not sure whether the GIT guys are happy for us to use this mailing
> list for StGIT. If the StGIT traffic increases, I will try to create a
> separate mailing list (maybe using a site like sf.net).
>

I'd very much like to stay on the same list. By the same logic, cogito should have it's own list as well...

Bryan
Jerry Seutter· Jul 13, 2005, 21:09 UTC · re: Bryan Larsen · lore

Re: [PATCH] stgit: allow spaces in filenames

Bryan Larsen wrote:
Show 9 quoted lines
> Catalin Marinas wrote:
> 
>> An additional thing, can you please convert all the tabs to spaces?
>> That's a better convention for a language like Python where you
>> delimit blocks by indentation.
> 
> 
> I would have hoped that emacs py-mode would "do the right thing". 
> Anybody know how to make it do what Catalin wants?

Yeah, the default emacs mode seems to be to Do The Wrong Thing. I have this in my .emacs, YMMV.

(setq-default indent-tabs-mode nil)     ; Don't insert tab characters.
(setq-default tab-width 4)              ; If there are tabs, display
                                         ; as 4 spaces.

You can set the tab-width to something much larger to make existing tabs obvious.

Jerry
Catalin Marinas· Jul 13, 2005, 21:26 UTC · re: Bryan Larsen · lore

Re: [PATCH] stgit: allow spaces in filenames

On Wed, 2005-07-13 at 14:17 -0400, Bryan Larsen wrote:
> Catalin Marinas wrote:
> I would have hoped that emacs py-mode would "do the right thing". 
> Anybody know how to make it do what Catalin wants?

It looks like the python-mode in my emacs does the right thing. You could add something like below in your .emacs file:

(add-hook 'python-mode-hook
          #'(lambda ()
              (setq indent-tabs-mode nil)))
Otherwise, select the whole buffer and do a "M-x untabify".
Show 5 quoted lines
> > What's the reason for having 'fin' as well? It doesn't seem to be used
> > (this is found in other parts of the patch as well).
> 
> popen does not support bypassing the shell by using vectors of 
> arguments.  Only popen2 and friends have this capability.

But the manual says that it is not possible to get the exit code of the child process with popen2 (at least not in python 2.3). You would need to use the Popen3 and Popen4 classes in the popen2 module.

Does it make that big difference if the commands are invoked via the shell? I haven't run any tests.

Show 7 quoted lines
> > I'm not sure whether the GIT guys are happy for us to use this mailing
> > list for StGIT. If the StGIT traffic increases, I will try to create a
> > separate mailing list (maybe using a site like sf.net).
> > 
> 
> I'd very much like to stay on the same list.  By the same logic, cogito 
> should have it's own list as well...

I'd like this too and it's probably OK with a low traffic (we'll see if we receive complaints :-) ).

Catalin
Junio C Hamano· Jul 13, 2005, 22:26 UTC · re: Catalin Marinas · lore

Re: [PATCH] stgit: allow spaces in filenames

Catalin Marinas <catalin.marinas@gmail.com> writes:
Show 5 quoted lines
>> I'd very much like to stay on the same list.  By the same logic, cogito 
>> should have it's own list as well...
>
> I'd like this too and it's probably OK with a low traffic (we'll see if
> we receive complaints :-) ).

I'd like to keep Porcelain discussions on this list for two reasons:

 (1) Porcelain implementations need their own bookkeeping
     information somewhere, and possibly in .git/ directory.
     I'd want to see them agree on what's stored where for what
     purpose and stay compatible when that makes sense.  The
     place to brew that concensus is here.
 (2) The core GIT people want to learn what Porcelain needs
     from the core.  I am personally interested in StGIT so
     subscribing to that list is fine for me, but it is
     convenient to have everything in one place, especially with
     this relatively low traffic.
Catalin Marinas· Jul 14, 2005, 07:02 UTC · re: Junio C Hamano · lore

Re: [PATCH] stgit: allow spaces in filenames

On Wed, 2005-07-13 at 15:26 -0700, Junio C Hamano wrote:
Show 10 quoted lines
> Catalin Marinas <catalin.marinas@gmail.com> writes:
> 
> >> I'd very much like to stay on the same list.  By the same logic, cogito 
> >> should have it's own list as well...
> >
> > I'd like this too and it's probably OK with a low traffic (we'll see if
> > we receive complaints :-) ).
> 
> I'd like to keep Porcelain discussions on this list for two
> reasons:
That's great then. I will keep the StGIT discussions on this list.

-- Catalin

Bryan Larsen· Jul 14, 2005, 07:22 UTC · re: Catalin Marinas · lore

Re: [PATCH] stgit: allow spaces in filenames

> Does it make that big difference if the commands are invoked via the
> shell? I haven't run any tests.
>

It wasn't for the time difference that I bypassed the shell, it was to support spaces and other strange characters in parameters. It's easy to use spawnvp than it is to escape the parameters properly.

Bryan
Matthias Urlichs· Jul 13, 2005, 16:54 UTC · re: Bryan Larsen · lore

Re: [PATCH] stgit: allow spaces in filenames

Hi, Bryan Larsen wrote:
> +	r=os.spawnvp(os.P_WAIT, args_l[0], args_l + args[i:min(i+1000, len(args))])

The max length for argv is 32k IIRC, so 1000 is 28-byte file names. That's probably not enough. I't suggest using chunks of 100.

-- 
Matthias Urlichs   |   {M:U} IT Design @ m-u-it.de   |  smurf@smurf.noris.de
Disclaimer: The quote was selected randomly. Really. | http://smurf.noris.de
 - -
Don't you feel more like you do now than you did when you came in?
Linus Torvalds· Jul 13, 2005, 19:51 UTC · re: Matthias Urlichs · lore

Re: [PATCH] stgit: allow spaces in filenames

On Wed, 13 Jul 2005, Matthias Urlichs wrote:
Show 6 quoted lines
>
> Hi, Bryan Larsen wrote:
> 
> > +	r=os.spawnvp(os.P_WAIT, args_l[0], args_l + args[i:min(i+1000, len(args))])
> 
> The max length for argv is 32k IIRC, so 1000 is 28-byte file names.

I think 32k may be the posix-mandated minimum. Linux does 128kB, so 1000 should likely be ok for most projects.

That said, xargs should do things like that right.
		Linus

← back to recent threads