{"thread":{"id":"999","subject":"[PATCH] fix scalability problems with git-deltafy-script (fwd)","startedAt":"2005-06-21T14:18:00Z","lastAt":"2005-06-21T14:18:00Z","messageCount":1,"participants":["Nicolas Pitre"],"isPatch":true,"patchVersion":1,"patchTotal":null},"messages":[{"id":"5098","messageId":"Pine.LNX.4.63.0506211008380.1667@localhost.localdomain","threadId":"999","inReplyTo":null,"subject":"[PATCH] fix scalability problems with git-deltafy-script (fwd)","fromName":"Nicolas Pitre","fromEmail":"nico@cam.org","sentAt":"2005-06-21T14:18:00Z","receivedAt":"2005-06-21T14:18:00Z","isPatch":true,"sender":{"key":"nico@fluxnic.net","avatar":"https://avatars.githubusercontent.com/u/702790?v=4"},"body":"\n[ Third resent... ]\n\nCurrent version would spin forever  and exhaust memory while \nattempting to sort all files from all revisions at once, until it\ndies before even doing any real work.  This is especially noticeable \nwhen used on a big repository like the imported  bkcvs repo for the\nLinux kernel.\n\nThis patch allows for batching the sort to put a bound on needed \nresources and making progress early, as well as including some small\ncleanups.\n\nSigned-off-by: Nicolas Pitre <nico@cam.org>\n\ndiff --git a/git-deltafy-script b/git-deltafy-script\n--- a/git-deltafy-script\n+++ b/git-deltafy-script\n@@ -1,6 +1,6 @@\n #!/bin/bash\n \n-# Example script to deltafy an entire GIT repository based on the commit list.\n+# Example script to deltify an entire GIT repository based on the commit list.\n # The most recent version of a file is the reference and previous versions\n # are made delta against the best earlier version available. And so on for\n # successive versions going back in time.  This way the increasing delta\n@@ -25,37 +25,51 @@\n \n set -e\n \n-depth=\n-[ \"$1\" == \"-d\" ] && depth=\"--max-depth=$2\" && shift 2\n+max_depth=\n+[ \"$1\" == \"-d\" ] && max_depth=\"--max-depth=$2\" && shift 2\n+\n+overlap=30\n+max_behind=\"--max-behind=$overlap\"\n \n function process_list() {\n \tif [ \"$list\" ]; then\n \t\techo \"Processing $curr_file\"\n-\t\techo \"$head $list\" | xargs git-mkdelta $depth --max-behind=30 -v\n+\t\techo \"$list\" | xargs git-mkdelta $max_depth $max_behind -v\n \tfi\n }\n \n+rev_list=\"\"\n curr_file=\"\"\n \n git-rev-list HEAD |\n-git-diff-tree -r -t --stdin |\n-awk '/^:/ { if ($5 == \"M\" || $5 == \"N\") print $4, $6;\n-            if ($5 == \"M\") print $3, $6 }' |\n-LC_ALL=C sort -s -k 2 | uniq |\n-while read sha1 file; do\n-\tif [ \"$file\" == \"$curr_file\" ]; then\n-\t\tlist=\"$list $sha1\"\n-\telse\n-\t\tprocess_list\n-\t\tcurr_file=\"$file\"\n-\t\tlist=\"\"\n-\t\thead=\"$sha1\"\n-\tfi\n+while true; do\n+\t# Let's batch revisions into groups of 1000 to give it a chance to\n+\t# scale with repositories containing long revision lists.  We also\n+\t# overlap with the previous batch the size of mkdelta's look behind\n+\t# value in order to account for the processing discontinuity.\n+\trev_list=\"$(echo -e -n \"$rev_list\" | tail --lines=$overlap)\"\n+\tfor i in $(seq 1000); do\n+\t\tread rev || break\n+\t\trev_list=\"$rev_list$rev\\n\"\n+\tdone\n+\techo -e -n \"$rev_list\" |\n+\tgit-diff-tree -r -t --stdin |\n+\tawk '/^:/ { if ($5 == \"M\") printf \"%s %s\\n%s %s\\n\", $4, $6, $3, $6 }' |\n+\tLC_ALL=C sort -s -k 2 | uniq |\n+\twhile read sha1 file; do\n+\t\tif [ \"$file\" == \"$curr_file\" ]; then\n+\t\t\tlist=\"$list $sha1\"\n+\t\telse\n+\t\t\tprocess_list\n+\t\t\tcurr_file=\"$file\"\n+\t\t\tlist=\"$sha1\"\n+\t\tfi\n+\tdone\n+\t[ \"$rev\" ] || break\n done\n process_list\n \n curr_file=\"root directory\"\n-head=\"\"\n list=\"$(\n \tgit-rev-list HEAD |\n \twhile read commit; do\n"}]}