# [PATCH] stgit: allow spaces in filenames 11 messages from 2005-07-13 to 2005-07-14. Participants: Bryan Larsen, Catalin Marinas, Matthias Urlichs, Linus Torvalds, Jerry Seutter, Junio C Hamano. Thread: https://gitlist.dev/t/1219 ## Bryan Larsen, 2005-07-13 08:38 Subject: [PATCH] stgit: allow spaces in filenames Message-ID: <20050713083805.18461.87079.sendpatchset@bryan-larsens-ibook-g4.local> URL: https://gitlist.dev/e/20050713083805.18461.87079.sendpatchset%40bryan-larsens-ibook-g4.local ``` The current version of stgit does not allow whitespace in filenames. This patch fixes that. It also speeds up operations on large filesets considerably. Signed-off-by: Bryan Larsen --- stgit/git.py | 158 +++++++++++++++++++++++++++++++-------------------------- stgit/main.py | 8 +-- 2 files changed, 90 insertions(+), 76 deletions(-) diff --git a/stgit/git.py b/stgit/git.py --- a/stgit/git.py +++ b/stgit/git.py @@ -101,6 +101,24 @@ def __output(cmd): raise GitException, '%s failed' % cmd return string +# __run: runs cmd using spawnvp. +# +# The shell is avoided so it won't mess up our arguments. +# If args is very large, the command is run multiple times; +# args is split xargs style: cmd is passed on each invocation. +# Unlike xargs, returns immediately if any non-zero return code +# is received. +# +def __run(cmd, args=None): + args_l=cmd.split() + if args is None: + args = [] + for i in range(0, len(args)+1, 1000): + r=os.spawnvp(os.P_WAIT, args_l[0], args_l + args[i:min(i+1000, len(args))]) + if r: + return r + return 0 + def __check_base_dir(): return os.path.isdir(base_dir) @@ -114,14 +132,16 @@ def __tree_status(files = [], tree_id = # unknown files if unknown: exclude_file = os.path.join(base_dir, 'exclude') - extra_exclude = '' + extra_exclude = [] if os.path.exists(exclude_file): - extra_exclude += ' --exclude-from=%s' % exclude_file - fout = os.popen('git-ls-files --others' - ' --exclude="*.[ao]" --exclude=".*"' - ' --exclude=TAGS --exclude=tags --exclude="*~"' - ' --exclude="#*"' + extra_exclude, 'r') + extra_exclude.append('--exclude-from=%s' % exclude_file) + fin, fout = os.popen2(['git-ls-files', '--others', + '--exclude=*.[ao]', '--exclude=.*' + '--exclude=TAGS', '--exclude=tags', '--exclude=*~', + '--exclude=#*'] + extra_exclude) cache_files += [('?', line.strip()) for line in fout] + fin.close() + fout.close() # conflicted files conflicts = get_conflicts() @@ -130,13 +150,12 @@ def __tree_status(files = [], tree_id = cache_files += [('C', filename) for filename in conflicts] # the rest - files_str = reduce(lambda x, y: x + ' ' + y, files, '') - fout = os.popen('git-diff-cache -r %s %s' % (tree_id, files_str), 'r') + fin, fout = os.popen2(['git-diff-cache', '-r', tree_id] + files) for line in fout: - fs = tuple(line.split()[4:]) + fs = tuple(line.rstrip().split(' ',4)[-1].split('\t',1)) if fs[1] not in conflicts: cache_files.append(fs) - if fout.close(): + if fout.close() or fin.close(): raise GitException, 'git-diff-cache failed' return cache_files @@ -185,10 +204,9 @@ def add(names): else: raise GitException, '%s is not a file or directory' % i - for f in files: - print 'Adding file %s' % f - if os.system('git-update-cache --add -- %s' % f) != 0: - raise GitException, 'Unable to add %s' % f + if files: + if __run('git-update-cache --add --', files): + raise GitException, 'Unable to add file' def rm(files, force = False): """Remove a file from the repository @@ -198,26 +216,32 @@ def rm(files, force = False): else: git_opt = '--remove' - for f in files: - if force: - print 'Removing file %s' % f - if os.system('git-update-cache --force-remove -- %s' % f) != 0: - raise GitException, 'Unable to remove %s' % f - elif os.path.exists(f): - raise GitException, '%s exists. Remove it first' %f - else: - print 'Removing file %s' % f - if os.system('git-update-cache --remove -- %s' % f) != 0: - raise GitException, 'Unable to remove %s' % f + if not force: + for f in files: + if os.path.exists(f): + raise GitException, '%s exists. Remove it first' %f + if files: + __run('git-update-cache --remove --', files) + else: + if files: + __run('git-update-cache --force-remove --', files) def update_cache(files): """Update the cache information for the given files """ + files_here = [] + files_gone = [] + for f in files: if os.path.exists(f): - os.system('git-update-cache -- %s' % f) + files_here.append(f) else: - os.system('git-update-cache --remove -- %s' % f) + files_gone.append(f) + + if files_here: + __run('git-update-cache --', files_here) + if files_gone: + __run('git-update-cache --remove --', files_gone) def commit(message, files = [], parents = [], allowempty = False, author_name = None, author_email = None, author_date = None, @@ -248,16 +272,26 @@ def commit(message, files = [], parents # update the cache if not first: + add_files=[] + rm_files=[] + m_files=[] for f in cache_files: if f[0] == 'N': - git_flag = '--add' + add_files.append(f[1]) elif f[0] == 'D': - git_flag = '--force-remove' + rm_files.append(f[1]) else: - git_flag = '--' + m_files.append(f[1]) - if os.system('git-update-cache %s %s' % (git_flag, f[1])) != 0: - raise GitException, 'Failed git-update-cache -- %s' % f[1] + if add_files: + if __run('git-update-cache --add --', add_files): + raise GitException, 'Failed git-update-cache --add' + if rm_files: + if __run('git-update-cache --force-remove --', rm_files): + raise GitException, 'Failed git-update-cache --rm' + if m_files: + if __run('git-update-cache --', m_files): + raise GitException, 'Failed git-update-cache' # write the index to repository tree_id = __output('git-write-tree') @@ -292,7 +326,7 @@ def merge(base, head1, head2): """Perform a 3-way merge between base, head1 and head2 into the local tree """ - if os.system('git-read-tree -u -m %s %s %s' % (base, head1, head2)) != 0: + if __run('git-read-tree -u -m', [base, head1, head2]) != 0: raise GitException, 'git-read-tree failed (local changes maybe?)' # this can fail if there are conflicts @@ -330,51 +364,31 @@ def status(files = [], modified = False, else: print '%s' % fs[1] -def diff(files = [], rev1 = 'HEAD', rev2 = None, output = None, - append = False): +def diff(files = [], rev1 = 'HEAD', rev2 = None, out_fd = sys.stdout): """Show the diff between rev1 and rev2 """ - files_str = reduce(lambda x, y: x + ' ' + y, files, '') - - extra_args = '' - if output: - if append: - extra_args += ' >> %s' % output - else: - extra_args += ' > %s' % output - os.system('git-update-cache --refresh > /dev/null') if rev2: - if os.system('git-diff-tree -p %s %s %s %s' - % (rev1, rev2, files_str, extra_args)) != 0: - raise GitException, 'git-diff-tree failed' + nf, in_fd = os.popen2(['git-diff-tree', '-p', rev1, rev2]+files) else: - if os.system('git-diff-cache -p %s %s %s' - % (rev1, files_str, extra_args)) != 0: - raise GitException, 'git-diff-cache failed' + nf, in_fd = os.popen2(['git-diff-cache', '-p', rev1]+files) + + out_fd.write(in_fd.read()) + if in_fd.close() or nf.close(): + raise GitException, 'git.diff failed' def diffstat(files = [], rev1 = 'HEAD', rev2 = None): """Return the diffstat between rev1 and rev2 """ - files_str = reduce(lambda x, y: x + ' ' + y, files, '') os.system('git-update-cache --refresh > /dev/null') - ds_cmd = '| git-apply --stat' - - if rev2: - f = os.popen('git-diff-tree -p %s %s %s %s' - % (rev1, rev2, files_str, ds_cmd), 'r') - str = f.read().rstrip() - if f.close(): - raise GitException, 'git-diff-tree failed' - else: - f = os.popen('git-diff-cache -p %s %s %s' - % (rev1, files_str, ds_cmd), 'r') - str = f.read().rstrip() - if f.close(): - raise GitException, 'git-diff-cache failed' - + ds_in, ds_out = os.popen2('git-apply --stat') + diff(files, rev1, rev2, ds_in) + ds_in.close() + str = ds_out.read().rstrip() + if ds_out.close(): + raise GitException, 'git.diffstat failed' return str def files(rev1, rev2): @@ -386,7 +400,7 @@ def files(rev1, rev2): f = os.popen('git-diff-tree -r %s %s' % (rev1, rev2), 'r') for line in f: - str += '%s %s\n' % tuple(line.split()[4:]) + str += '%s %s\n' % tuple(line.rstrip().split(' ',4)[-1].split('\t',1)) if f.close(): raise GitException, 'git-diff-tree failed' @@ -395,23 +409,23 @@ def files(rev1, rev2): def checkout(files = [], force = False): """Check out the given or all files """ - git_flags = '' + git_flags = 'git-checkout-cache -q -u' if force: git_flags += ' -f' if len(files) == 0: git_flags += ' -a' else: - git_flags += reduce(lambda x, y: x + ' ' + y, files, ' --') + git_flags += ' --' - if os.system('git-checkout-cache -q -u%s' % git_flags) != 0: - raise GitException, 'Failed git-checkout-cache -q -u%s' % git_flags + if __run(git_flags, files) != 0: + raise GitException, 'Failed git-checkout-cache' def switch(tree_id): """Switch the tree to the given id """ to_delete = filter(lambda x: x[0] == 'N', __tree_status(tree_id = tree_id)) - if os.system('git-read-tree -m %s' % tree_id) != 0: + if __run('git-read-tree -m', [tree_id]) != 0: raise GitException, 'Failed git-read-tree -m %s' % tree_id checkout(force = True) diff --git a/stgit/main.py b/stgit/main.py --- a/stgit/main.py +++ b/stgit/main.py @@ -720,13 +720,13 @@ def export(parser, options, args): 'supported in the patch template' f = open(pfile, 'w+') f.write(descr) - f.close() # write the diff - git.diff(rev1 = __git_id('%s/bottom' % p), + git.diff(rev1 = __git_id('%s/bottom' % p), rev2 = __git_id('%s/top' % p), - output = pfile, append = True) - patch_no += 1 + out_fd = f) + f.close() + patch_no += 1 series.close() ``` ## Bryan Larsen, 2005-07-13 08:38 Subject: [PATCH] stgit: new --message does not work Message-ID: <20050713083816.18461.32180.sendpatchset@bryan-larsens-ibook-g4.local> URL: https://gitlist.dev/e/20050713083816.18461.32180.sendpatchset%40bryan-larsens-ibook-g4.local In-Reply-To: <20050713083805.18461.87079.sendpatchset@bryan-larsens-ibook-g4.local> ``` Stacked GIT: Using the --message option with the new command fails. Fix it. Signed-off-by: Bryan Larsen --- stgit/stack.py | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/stgit/stack.py b/stgit/stack.py --- a/stgit/stack.py +++ b/stgit/stack.py @@ -384,6 +384,8 @@ class Series: descr = edit_file(None, \ 'Please enter the description for patch "%s" ' \ 'above.' % name) + else: + descr = message head = git.get_head() ``` ## Catalin Marinas, 2005-07-13 10:17 Subject: Re: [PATCH] stgit: allow spaces in filenames Message-ID: URL: https://gitlist.dev/e/tnxll4bdn7b.fsf%40arm.com In-Reply-To: <20050713083805.18461.87079.sendpatchset@bryan-larsens-ibook-g4.local> ``` Bryan Larsen wrote: > The current version of stgit does not allow whitespace in filenames. > This patch fixes that. It also speeds up operations on large > filesets considerably. Thanks, I will apply it but I have a few comments below: > +# __run: runs cmd using spawnvp. > +# > +# The shell is avoided so it won't mess up our arguments. > +# If args is very large, the command is run multiple times; > +# args is split xargs style: cmd is passed on each invocation. > +# Unlike xargs, returns immediately if any non-zero return code > +# is received. > +# > +def __run(cmd, args=None): I would prefer to add this as Python function documentation, i.e. with """...""" in the function body. An additional thing, can you please convert all the tabs to spaces? That's a better convention for a language like Python where you delimit blocks by indentation. > @@ -114,14 +132,16 @@ def __tree_status(files = [], tree_id = > # unknown files > if unknown: > exclude_file = os.path.join(base_dir, 'exclude') > - extra_exclude = '' > + extra_exclude = [] > if os.path.exists(exclude_file): > - extra_exclude += ' --exclude-from=%s' % exclude_file > - fout = os.popen('git-ls-files --others' > - ' --exclude="*.[ao]" --exclude=".*"' > - ' --exclude=TAGS --exclude=tags --exclude="*~"' > - ' --exclude="#*"' + extra_exclude, 'r') > + extra_exclude.append('--exclude-from=%s' % exclude_file) > + fin, fout = os.popen2(['git-ls-files', '--others', > + '--exclude=*.[ao]', '--exclude=.*' > + '--exclude=TAGS', '--exclude=tags', '--exclude=*~', > + '--exclude=#*'] + extra_exclude) > cache_files += [('?', line.strip()) for line in fout] > + fin.close() > + fout.close() What's the reason for having 'fin' as well? It doesn't seem to be used (this is found in other parts of the patch as well). Something wrong happened to my git-ftp-push script. It looks like it copied 'master' to stgit.git/ and not to stgit.git/refs/heads/. I can't fix it until tonight. Before then, you could actually use wget to pull the whole repository and just copy 'master' to 'refs/heads/master'. In the latest tree I created separate files for each command. I'm not sure whether the GIT guys are happy for us to use this mailing list for StGIT. If the StGIT traffic increases, I will try to create a separate mailing list (maybe using a site like sf.net). -- Catalin ``` ## Matthias Urlichs, 2005-07-13 16:54 Subject: Re: [PATCH] stgit: allow spaces in filenames Message-ID: URL: https://gitlist.dev/e/pan.2005.07.13.16.54.18.709929%40smurf.noris.de In-Reply-To: <20050713083805.18461.87079.sendpatchset@bryan-larsens-ibook-g4.local> ``` Hi, Bryan Larsen wrote: > + r=os.spawnvp(os.P_WAIT, args_l[0], args_l + args[i:min(i+1000, len(args))]) The max length for argv is 32k IIRC, so 1000 is 28-byte file names. That's probably not enough. I't suggest using chunks of 100. -- Matthias Urlichs | {M:U} IT Design @ m-u-it.de | smurf@smurf.noris.de Disclaimer: The quote was selected randomly. Really. | http://smurf.noris.de - - Don't you feel more like you do now than you did when you came in? ``` ## Bryan Larsen, 2005-07-13 18:17 Subject: Re: [PATCH] stgit: allow spaces in filenames Message-ID: <42D55AC9.4000506@gmail.com> URL: https://gitlist.dev/e/42D55AC9.4000506%40gmail.com In-Reply-To: ``` Catalin Marinas wrote: > An additional thing, can you please convert all the tabs to spaces? > That's a better convention for a language like Python where you > delimit blocks by indentation. I would have hoped that emacs py-mode would "do the right thing". Anybody know how to make it do what Catalin wants? > > >>+ fin, fout = os.popen2(['git-ls-files', '--others', > > > What's the reason for having 'fin' as well? It doesn't seem to be used > (this is found in other parts of the patch as well). popen does not support bypassing the shell by using vectors of arguments. Only popen2 and friends have this capability. Unfortunate, yes. > > I'm not sure whether the GIT guys are happy for us to use this mailing > list for StGIT. If the StGIT traffic increases, I will try to create a > separate mailing list (maybe using a site like sf.net). > I'd very much like to stay on the same list. By the same logic, cogito should have it's own list as well... Bryan ``` ## Linus Torvalds, 2005-07-13 19:51 Subject: Re: [PATCH] stgit: allow spaces in filenames Message-ID: URL: https://gitlist.dev/e/Pine.LNX.4.58.0507131250060.17536%40g5.osdl.org In-Reply-To: ``` On Wed, 13 Jul 2005, Matthias Urlichs wrote: > > Hi, Bryan Larsen wrote: > > > + r=os.spawnvp(os.P_WAIT, args_l[0], args_l + args[i:min(i+1000, len(args))]) > > The max length for argv is 32k IIRC, so 1000 is 28-byte file names. I think 32k may be the posix-mandated minimum. Linux does 128kB, so 1000 should likely be ok for most projects. That said, xargs should do things like that right. Linus ``` ## Jerry Seutter, 2005-07-13 21:09 Subject: Re: [PATCH] stgit: allow spaces in filenames Message-ID: <42D58324.6080303@pason.com> URL: https://gitlist.dev/e/42D58324.6080303%40pason.com In-Reply-To: <42D55AC9.4000506@gmail.com> ``` Bryan Larsen wrote: > Catalin Marinas wrote: > >> An additional thing, can you please convert all the tabs to spaces? >> That's a better convention for a language like Python where you >> delimit blocks by indentation. > > > I would have hoped that emacs py-mode would "do the right thing". > Anybody know how to make it do what Catalin wants? Yeah, the default emacs mode seems to be to Do The Wrong Thing. I have this in my .emacs, YMMV. (setq-default indent-tabs-mode nil) ; Don't insert tab characters. (setq-default tab-width 4) ; If there are tabs, display ; as 4 spaces. You can set the tab-width to something much larger to make existing tabs obvious. Jerry ``` ## Catalin Marinas, 2005-07-13 21:26 Subject: Re: [PATCH] stgit: allow spaces in filenames Message-ID: <1121290004.6876.11.camel@localhost.localdomain> URL: https://gitlist.dev/e/1121290004.6876.11.camel%40localhost.localdomain In-Reply-To: <42D55AC9.4000506@gmail.com> ``` On Wed, 2005-07-13 at 14:17 -0400, Bryan Larsen wrote: > Catalin Marinas wrote: > I would have hoped that emacs py-mode would "do the right thing". > Anybody know how to make it do what Catalin wants? It looks like the python-mode in my emacs does the right thing. You could add something like below in your .emacs file: (add-hook 'python-mode-hook #'(lambda () (setq indent-tabs-mode nil))) Otherwise, select the whole buffer and do a "M-x untabify". > > What's the reason for having 'fin' as well? It doesn't seem to be used > > (this is found in other parts of the patch as well). > > popen does not support bypassing the shell by using vectors of > arguments. Only popen2 and friends have this capability. But the manual says that it is not possible to get the exit code of the child process with popen2 (at least not in python 2.3). You would need to use the Popen3 and Popen4 classes in the popen2 module. Does it make that big difference if the commands are invoked via the shell? I haven't run any tests. > > I'm not sure whether the GIT guys are happy for us to use this mailing > > list for StGIT. If the StGIT traffic increases, I will try to create a > > separate mailing list (maybe using a site like sf.net). > > > > I'd very much like to stay on the same list. By the same logic, cogito > should have it's own list as well... I'd like this too and it's probably OK with a low traffic (we'll see if we receive complaints :-) ). Catalin ``` ## Junio C Hamano, 2005-07-13 22:26 Subject: Re: [PATCH] stgit: allow spaces in filenames Message-ID: <7vpstms5pc.fsf@assigned-by-dhcp.cox.net> URL: https://gitlist.dev/e/7vpstms5pc.fsf%40assigned-by-dhcp.cox.net In-Reply-To: <1121290004.6876.11.camel@localhost.localdomain> ``` Catalin Marinas writes: >> I'd very much like to stay on the same list. By the same logic, cogito >> should have it's own list as well... > > I'd like this too and it's probably OK with a low traffic (we'll see if > we receive complaints :-) ). I'd like to keep Porcelain discussions on this list for two reasons: (1) Porcelain implementations need their own bookkeeping information somewhere, and possibly in .git/ directory. I'd want to see them agree on what's stored where for what purpose and stay compatible when that makes sense. The place to brew that concensus is here. (2) The core GIT people want to learn what Porcelain needs from the core. I am personally interested in StGIT so subscribing to that list is fine for me, but it is convenient to have everything in one place, especially with this relatively low traffic. ``` ## Catalin Marinas, 2005-07-14 07:02 Subject: Re: [PATCH] stgit: allow spaces in filenames Message-ID: <1121324523.6873.1.camel@localhost.localdomain> URL: https://gitlist.dev/e/1121324523.6873.1.camel%40localhost.localdomain In-Reply-To: <7vpstms5pc.fsf@assigned-by-dhcp.cox.net> ``` On Wed, 2005-07-13 at 15:26 -0700, Junio C Hamano wrote: > Catalin Marinas writes: > > >> I'd very much like to stay on the same list. By the same logic, cogito > >> should have it's own list as well... > > > > I'd like this too and it's probably OK with a low traffic (we'll see if > > we receive complaints :-) ). > > I'd like to keep Porcelain discussions on this list for two > reasons: That's great then. I will keep the StGIT discussions on this list. -- Catalin ``` ## Bryan Larsen, 2005-07-14 07:22 Subject: Re: [PATCH] stgit: allow spaces in filenames Message-ID: <42D612BB.5040400@gmail.com> URL: https://gitlist.dev/e/42D612BB.5040400%40gmail.com In-Reply-To: <1121290004.6876.11.camel@localhost.localdomain> ``` > Does it make that big difference if the commands are invoked via the > shell? I haven't run any tests. > It wasn't for the time difference that I bypassed the shell, it was to support spaces and other strange characters in parameters. It's easy to use spawnvp than it is to escape the parameters properly. Bryan ```