git/list[1] front-page[2] threads[3] people[4] search[5] about
 

Re: RFC: Sparse checkout improvements (was: Re: question (possibly) on git subtree/submodules)

From
Sskillzero@gmail.com <skillzero@gmail.com>
Date
Jul 27, 2010, 16:55 UTC
Message-ID
<AANLkTimJpmGqOogQ1HtN0zu0=tPK0C0kQuEkg4PeOqUh@mail.gmail.com>
In-Reply-To
<4C4EEC0D.2070105@xiplink.com>
On Tue, Jul 27, 2010 at 7:24 AM, Marc Branchaud <marcnarc@xiplink.com> wrote:
> * What's missing is a way to define named collections of paths
> ("sparse-sets?") in .git/info/sparse-checkout, so that you can conveniently
> checkout a particular subset of the working directory.  It would also be nice
> to switch between different sparse-sets.

I pasted in a script I wrote to work with the sparse checkout feature. I'm not a scripting expert so it probably doesn't things incorrectly. It lets you create "modules" by adding sections to .gitmodules file at the root of the repository (or a file you specify). You can list them, switch/checkout between them, or reset back to normal:

[module "MyApp1"]
	<path1>
	<path2>

$ git module list MyApp1

$ git module checkout MyApp1
$ git module reset
> * It would also be good to have a way for a repo to define a default
> sparse-set, so that a clone would only checkout that default.

Yes, this would be nice. Ideally what I would like is for there to be a clone option to specify a "module" (what I've been calling sparse sets). A first step could just clone the full repository with -n then do 'git module checkout <module>' (what my other scripts do to prepare archives). Ideally, it would do some work on the server side to only send the objects needed for paths specified by the sparse set (but still allow me to commit and later push changes back).

-- 
git-module script (email may mess up the spacing, causing things to
not line up, but you get the idea)

use Getopt::Long qw(:config gnu_getopt);
use File::Path;

my $gBranch			= "";
my $gHelp			= 0;
my $gModulesFile	= "";
my $gModules		= {};
my $gRecursive		= 0;

# Parse the command line.
if( @ARGV < 1 ) { Usage(); }
GetOptions(
	"b|branch=s"		=> \$gBranch,
	"h|help"			=> \$gHelp,
	"f|modules-file=s"	=> \$gModulesFile,
	"r|recursive!"		=> \$gRecursive,
) or die( "\n" );
if( $gHelp ) { Usage(); }

if( @ARGV < 1 ) { die( "error: no command specified. See 'git module
help'.\n" ); }
my $cmd = shift;
if(    $cmd eq "checkout" )	{ cmd_checkout(); }
elsif( $cmd eq "help" )		{ Usage(); }
elsif( $cmd eq "list" )		{ cmd_list(); }
elsif( $cmd eq "reset" )	{ cmd_reset(); }
else { die( "error: unknown command '$cmd'. See 'git module help'.\n" ); }

#
# cmd_checkout
#
sub	cmd_checkout
{
	ReadModulesFile();
	my $moduleName = shift @ARGV;
	if( !$moduleName )				{ die( "error: no module name specified.\n" ); }
	if( !$gModules->{$moduleName} )	{ die( "error: module '$moduleName'
not found.\n" ); }
	
	# Enable sparse.
	my $currentCmd = "git config core.sparseCheckout true";
	system( $currentCmd ) == 0 or die( "error: '$currentCmd' $?.\n" );
	
	# Write the sparse patterns.
	my $gitDir = `git rev-parse --git-dir`;
	if( $? != 0 ) { die( "error: can't find git repository $?.\n" ); }
	chop( $gitDir );
	my $sparsePath = $gitDir . "/info/sparse-checkout";
	if( $? != 0 ) { die( "error: read git directory failed $?.\n" ); }
	open( FILE, ">", $sparsePath ) or die( "error: can't open '$sparsePath'.\n" );
	foreach( @{$gModules->{$moduleName}} )
	{
		print( FILE "$_\n" );
	}
	close( FILE );
	
	# Checkout using the new sparse patterns.
	my $currentCmd = "git checkout $gBranch --";
	system( $currentCmd ) == 0 or die( "error: '$currentCmd' $?.\n" );
}

#
# cmd_list
#
sub	cmd_list
{
	ReadModulesFile();
	my $moduleName = shift @ARGV;
	if( $moduleName eq "" )
	{
		if( $gRecursive )
		{
			foreach my $moduleName ( sort( keys %{$gModules} ) )
			{
				print( "$moduleName\n" );
				foreach( @{$gModules->{$moduleName}} )
				{
					print( "\t$_\n" );
				}
			}
		}
		else
		{
			foreach my $moduleName ( sort( keys %{$gModules} ) )
			{
				print( "$moduleName\n" );
			}
		}
	}
	else
	{
		if( !$gModules->{$moduleName} ) { die( "module '$moduleName' not
found.\n" ); }
		foreach( @{$gModules->{$moduleName}} )
		{
			print( "$_\n" );
		}
	}
}

#
# cmd_reset
#
sub	cmd_reset
{
	# Enable sparse.
	my $currentCmd = "git config core.sparseCheckout true";
	system( $currentCmd ) == 0 or die( "error: '$currentCmd' $?.\n" );
	
	# Write a special sparse pattern of "*" to mean everything.
	my $gitDir = `git rev-parse --git-dir`;
	if( $? != 0 ) { die( "error: can't find git repository $?.\n" ); }
	chop( $gitDir );
	my $sparsePath = $gitDir . "/info/sparse-checkout";
	if( $? != 0 ) { die( "error: read git directory failed $?.\n" ); }
	open( FILE, ">", $sparsePath ) or die( "error: can't open '$sparsePath'.\n" );
	print( FILE "*\n" );
	close( FILE );
	
	# Checkout to clear the skip-worktree bits and checkout all entries.
	my $currentCmd = "git checkout $gBranch --";
	system( $currentCmd ) == 0 or die( "error: '$currentCmd' $?.\n" );
	
	# Disable sparse.
	my $currentCmd = "git config core.sparseCheckout false";
	system( $currentCmd ) == 0 or die( "error: '$currentCmd' $?.\n" );
}

#
# ReadModulesFile
#
sub	ReadModulesFile
{
	my @lines = ();
	if( $gModulesFile eq "" )		# No file means read from the .gitmodules
file in the repo.
	{
		if( $gBranch ne "" )	{ @lines = `git show $gBranch:.gitmodules`;}
		else					{ @lines = `git show HEAD:.gitmodules`; }
		if( $? != 0 )			{ die( "error: read .gitmodules file failed: $?.\n" ); }
	}
	elsif( $gModulesFile eq "-" )	# - means read from stdin.
	{
		@lines = <STDIN>;
	}
	else
	{
		open( FILE, $gModulesFile ) or die( "error: can't open '$gModulesFile'.\n" );
		@lines = <FILE>;
		close( FILE );
	}
	chomp( @lines );
	
	my $isModule   = 0;
	my $moduleName = "";
	foreach my $line ( @lines )
	{
		$line =~ s/^\s+//; # Strip leading whitespace.
		$line =~ s/\s+$//; # Strip trailing whitespace.
		$line =~ s/\r//g;  # Strip CR's.
		$line =~ s/\n//g;  # Strip LF's.
		if( $line =~ /\s*\[(.*?)\]\s*/ ) # Check for section header.
		{
			$moduleName = $1;
			if( $moduleName =~ /\s*module\s*\"(.*)\"/ )
			{
				$moduleName = $1;
				$isModule = 1;
			}
			else
			{
				$isModule = 0;
			}
			next;
		}
		next if !$isModule;			# Skip entries that aren't in module sections.
		next if $line =~ /^\s*\;/;	# Skip lines beginning with ';'.
		next if $line =~ /^\s*\#/;	# Skip lines beginning with '#'.
		next if length $line == 0;	# Skip empty lines.
		push( @{$gModules->{$moduleName}}, $line );
	}
}

#
# Usage
#
sub	Usage
{
	print( STDERR "Usage: git-module [options] command [command options]\n" );
	print( STDERR "\n" );
	print( STDERR "Options:\n" );
	print( STDERR "    -b/--branch <name>          Branch to use.\n" );
	print( STDERR "    -f/--modules-file <file>    Custom modules file to
use.\n" );
	print( STDERR "\n" );
	print( STDERR "Commands:\n" );
	print( STDERR "    checkout <name>     Check out a module.\n" );
	print( STDERR "    list [-r] [name]    List module(s). -r lists
modules and patterns.\n" );
	print( STDERR "    reset               Reset to a non-sparse checkout.\n" );
	print( STDERR "\n" );
	exit( 1 );
}
Previous: Marc BranchaudNext: Marc Branchaud
Message 8 of 9 in “question (possibly) on git subtree/submodules”
  1. Maurizio VitaleJul 23, 2010
  2. Chris PackhamJul 23, 2010
  3. Jonathan NiederJul 23, 2010
  4. Chris PackhamJul 23, 2010
  5. AlexJul 27, 2010
  6. Jakub NarebskiJul 27, 2010
  7. RFC: Sparse checkout improvements (was: Re: question (possibly) on git subtree/submodules)Marc Branchaud, Jul 27, 2010
  8. skillzero@gmail.comJul 27, 2010
  9. Marc BranchaudJul 28, 2010

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.