{"thread":{"id":"20085","subject":"[PATCH/RFC v2] git-p4: stream from perforce to speed up clones","startedAt":"2009-07-11T08:31:11Z","lastAt":"2009-07-25T14:25:09Z","messageCount":7,"participants":["Luke Diamand","Pete Wyckoff"],"isPatch":true,"patchVersion":2,"patchTotal":null},"messages":[{"id":"117818","messageId":"1247301071-9541-1-git-send-email-luke@diamand.org","threadId":"20085","inReplyTo":null,"subject":"[PATCH/RFC v2] git-p4: stream from perforce to speed up clones","fromName":"Luke Diamand","fromEmail":"luke@diamand.org","sentAt":"2009-07-11T08:31:11Z","receivedAt":"2009-07-11T08:31:11Z","isPatch":true,"sender":{"key":"luke@diamand.org","avatar":"https://avatars.githubusercontent.com/u/5330967?v=4"},"body":"Change commit() to stream data from Perforce and into fast-import\nrather than reading into memory first, and then writing out. This\nhugely reduces the memory requirements when cloning non-incrementally.\n\nSigned-off-by: Luke Diamand <luke@diamand.org>\n---\nI've modified git-p4 so that it streams/pipes data into fast-import rather\nthat reading everything into memory first. The old scheme meant that\nfor a large repository (mine is around 2G) my PC just grinds to a\nhalt and never actually finishes. With this change it takes around ten\nminutes.\n\nThis is a resend of a patch I sent earlier, which my MUA helpfully\nmanaged to word-wrap.\n\n contrib/fast-import/git-p4 |  164 ++++++++++++++++++++++++++++++++++++--------\n 1 files changed, 136 insertions(+), 28 deletions(-)\n\ndiff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4\nindex 342529d..f415ad0 100755\n--- a/contrib/fast-import/git-p4\n+++ b/contrib/fast-import/git-p4\n@@ -1008,6 +1008,141 @@ class P4Sync(Command):\n \n         return filesForCommit\n \n+    # output one file from the P4 stream\n+    # - helper for streamP4Files\n+\n+    def streamOneP4File(self, file, contents, branchPrefixes):\n+        if verbose:\n+            sys.stderr.write(\"%s\\n\" %  file[\"depotFile\"])\n+\n+        relPath = self.stripRepoPath(file['depotFile'], branchPrefixes)\n+\n+        mode = \"644\"\n+        if isP4Exec(file[\"type\"]):\n+            mode = \"755\"\n+        elif file[\"type\"] == \"symlink\":\n+            mode = \"120000\"\n+            # p4 print on a symlink contains \"target\\n\", so strip it off\n+            last = contents.pop()\n+            last = last[:-1]\n+            contents.append(last)\n+\n+        if self.isWindows and file[\"type\"].endswith(\"text\"):\n+            mangled = []\n+            for data in contents:\n+                data = data.replace(\"\\r\\n\", \"\\n\")\n+                mangled.append(data)\n+            contents = mangled\n+\n+        if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):\n+            contents = map(lambda text: re.sub(r'(?i)\\$(Id|Header):[^$]*\\$',r'$\\1$', text), contents)\n+        elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):\n+            contents = map(lambda text: re.sub(r'\\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\\n]*\\$',r'$\\1$', text), contents)\n+\n+        self.gitStream.write(\"M %s inline %s\\n\" % (mode, relPath))\n+\n+        # total length...\n+        length = 0\n+        for d in contents:\n+            length = length + len(d)\n+\n+        self.gitStream.write(\"data %d\\n\" % length)\n+        for d in contents:\n+            self.gitStream.write(d)\n+        self.gitStream.write(\"\\n\")\n+\n+    def streamOneP4Deletion(self, file, branchPrefixes):\n+        if verbose:\n+            sys.stderr.write(\"delete %s\\n\" %  file[\"path\"])\n+\n+        relPath = self.stripRepoPath(file['path'], branchPrefixes)\n+\n+        self.gitStream.write(\"D %s\\n\" % relPath)\n+\n+    # Stream directly from \"p4 files\" into \"git fast-import\"\n+    def streamP4Files(self, files, branchPrefixes):\n+        filesForCommit = []\n+        filesToRead = []\n+        filesToDelete = []\n+\n+        for f in files:\n+            includeFile = True\n+            for val in self.clientSpecDirs:\n+                if f['path'].startswith(val[0]):\n+                    if val[1] <= 0:\n+                        includeFile = False\n+                    break\n+\n+            if includeFile:\n+                filesForCommit.append(f)\n+                if f['action'] not in ('delete', 'purge'):\n+                    filesToRead.append(f)\n+                else:\n+                    filesToDelete.append(f)\n+\n+        filedata = []\n+\n+        # deleted files...\n+        for f in filesToDelete:\n+            self.streamOneP4Deletion(f, branchPrefixes)\n+\n+        if len(filesToRead) > 0:\n+            stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode='w+b')\n+            stdin_file.write('\\n'.join(['%s#%s' % (f['path'], f['rev'])\n+                                                  for f in filesToRead]))\n+            stdin_file.flush()\n+            stdin_file.seek(0)\n+            try:\n+                p4 = subprocess.Popen('p4 -G -x - print',\n+                                        shell=True,\n+                                        stdin=stdin_file,\n+                                        stdout=subprocess.PIPE);\n+            except OSError,e:\n+                print >> sys.stderr, \"p4 print failed:\", e\n+\n+            file = {}\n+            contents = []\n+            have_file_info = False\n+\n+            try:\n+                while True:\n+                    marshalled = marshal.load(p4.stdout)\n+\n+                    if marshalled.has_key('depotFile') and have_file_info:\n+                        # start of a new file - output the old one first\n+\n+                        if file[\"type\"] == \"apple\":\n+                            print \"\\nfile %s is a strange apple file that forks. Ignoring\" % file['path']\n+                            continue\n+\n+\n+                        self.streamOneP4File(file,contents,branchPrefixes)\n+                        file = {}\n+                        contents = []\n+                        have_file_info = False\n+\n+                    # pick up the new file information... for the\n+                    # 'data' field we need to append to our array\n+                    for k in marshalled.keys():\n+                        if k == 'data':\n+                            contents.append(marshalled['data'])\n+                        else:\n+                            file[k] = marshalled[k]\n+\n+                    have_file_info = True\n+            except EOFError:\n+                pass\n+\n+            # do the last chunk\n+\n+            if file.has_key('depotFile'):\n+                self.streamOneP4File(file,contents,branchPrefixes)\n+\n+            exitCode = p4.wait()\n+            if exitCode != 0:\n+                sys.stderr.write(\"p4 subshell failed getting file data\\n\")\n+                sys.exit(1)\n+\n     def commit(self, details, files, branch, branchPrefixes, parent = \"\"):\n         epoch = details[\"time\"]\n         author = details[\"user\"]\n@@ -1023,7 +1158,6 @@ class P4Sync(Command):\n                 new_files.append (f)\n             else:\n                 sys.stderr.write(\"Ignoring file outside of prefix: %s\\n\" % path)\n-        files = self.readP4Files(new_files)\n \n         self.gitStream.write(\"commit %s\\n\" % branch)\n #        gitStream.write(\"mark :%s\\n\" % details[\"change\"])\n@@ -1051,33 +1185,7 @@ class P4Sync(Command):\n                 print \"parent %s\" % parent\n             self.gitStream.write(\"from %s\\n\" % parent)\n \n-        for file in files:\n-            if file[\"type\"] == \"apple\":\n-                print \"\\nfile %s is a strange apple file that forks. Ignoring!\" % file['path']\n-                continue\n-\n-            relPath = self.stripRepoPath(file['path'], branchPrefixes)\n-            if file[\"action\"] in (\"delete\", \"purge\"):\n-                self.gitStream.write(\"D %s\\n\" % relPath)\n-            else:\n-                data = file['data']\n-\n-                mode = \"644\"\n-                if isP4Exec(file[\"type\"]):\n-                    mode = \"755\"\n-                elif file[\"type\"] == \"symlink\":\n-                    mode = \"120000\"\n-                    # p4 print on a symlink contains \"target\\n\", so strip it off\n-                    data = data[:-1]\n-\n-                if self.isWindows and file[\"type\"].endswith(\"text\"):\n-                    data = data.replace(\"\\r\\n\", \"\\n\")\n-\n-                self.gitStream.write(\"M %s inline %s\\n\" % (mode, relPath))\n-                self.gitStream.write(\"data %s\\n\" % len(data))\n-                self.gitStream.write(data)\n-                self.gitStream.write(\"\\n\")\n-\n+        self.streamP4Files(new_files,branchPrefixes)\n         self.gitStream.write(\"\\n\")\n \n         change = int(details[\"change\"])\n-- \n1.6.3.GIT\n"},{"id":"118723","messageId":"20090725142329.GA3841@arf.padd.com","threadId":"20085","inReplyTo":"1247301071-9541-1-git-send-email-luke@diamand.org","subject":"Re: [PATCH/RFC v2] git-p4: stream from perforce to speed up clones","fromName":"Pete Wyckoff","fromEmail":"pw@padd.com","sentAt":"2009-07-25T14:23:29Z","receivedAt":"2009-07-25T14:23:29Z","isPatch":true,"sender":{"key":"pw@padd.com","avatar":null},"body":"luke@diamand.org wrote on Sat, 11 Jul 2009 09:31 +0100:\n> Change commit() to stream data from Perforce and into fast-import\n> rather than reading into memory first, and then writing out. This\n> hugely reduces the memory requirements when cloning non-incrementally.\n> \n> Signed-off-by: Luke Diamand <luke@diamand.org>\n> ---\n> I've modified git-p4 so that it streams/pipes data into fast-import rather\n> that reading everything into memory first. The old scheme meant that\n> for a large repository (mine is around 2G) my PC just grinds to a\n> halt and never actually finishes. With this change it takes around ten\n> minutes.\n> \n> This is a resend of a patch I sent earlier, which my MUA helpfully\n> managed to word-wrap.\n\nThis is definitely good stuff.  While I'd like to get Daniel's\nnative transport code working in my environment, I had a need to\nimport some big trees quickly from p4 to git.  Usually I resort to a\nrare big-memory machine to do the git-p4 import, but with your\nchange, I can use my laptop without going into swap or\nout-of-memory.\n\nFor the content, I would recommend some changes.  Here are some\npatches on top of yours:\n\n    [PATCH 1/5] git-p4 stream: remove unused function\n    [PATCH 2/5] git-p4 stream: do not pass branchPrefixes so much\n    [PATCH 3/5] git-p4 stream: show relative path in debug messages\n    [PATCH 4/5] git-p4 stream: check apple file type\n    [PATCH 5/5] git-p4 stream: use existing p4CmdList with callback\n\nThey clean up the code and change your main iterator loop to reuse\nthe existing p4CmdList.\n\nI'd suggest you merge these into your patch and send it back out.\nIt's a worthwhile change and we should get Simon to review it.\n\n\t\t-- Pete\n"},{"id":"118721","messageId":"20090725142357.GB3841@arf.padd.com","threadId":"20085","inReplyTo":"20090725142329.GA3841@arf.padd.com","subject":"[PATCH 1/5] git-p4 stream: remove unused function","fromName":"Pete Wyckoff","fromEmail":"pw@padd.com","sentAt":"2009-07-25T14:23:57Z","receivedAt":"2009-07-25T14:23:57Z","isPatch":true,"sender":{"key":"pw@padd.com","avatar":null},"body":"This functionality was replaced.\n\nSigned-off-by: Pete Wyckoff <pw@padd.com>\n---\n contrib/fast-import/git-p4 |   58 --------------------------------------------\n 1 files changed, 0 insertions(+), 58 deletions(-)\n\ndiff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4\nindex f415ad0..70fa403 100755\n--- a/contrib/fast-import/git-p4\n+++ b/contrib/fast-import/git-p4\n@@ -950,64 +950,6 @@ class P4Sync(Command):\n \n         return branches\n \n-    ## Should move this out, doesn't use SELF.\n-    def readP4Files(self, files):\n-        filesForCommit = []\n-        filesToRead = []\n-\n-        for f in files:\n-            includeFile = True\n-            for val in self.clientSpecDirs:\n-                if f['path'].startswith(val[0]):\n-                    if val[1] <= 0:\n-                        includeFile = False\n-                    break\n-\n-            if includeFile:\n-                filesForCommit.append(f)\n-                if f['action'] not in ('delete', 'purge'):\n-                    filesToRead.append(f)\n-\n-        filedata = []\n-        if len(filesToRead) > 0:\n-            filedata = p4CmdList('-x - print',\n-                                 stdin='\\n'.join(['%s#%s' % (f['path'], f['rev'])\n-                                                  for f in filesToRead]),\n-                                 stdin_mode='w+')\n-\n-            if \"p4ExitCode\" in filedata[0]:\n-                die(\"Problems executing p4. Error: [%d].\"\n-                    % (filedata[0]['p4ExitCode']));\n-\n-        j = 0;\n-        contents = {}\n-        while j < len(filedata):\n-            stat = filedata[j]\n-            j += 1\n-            text = ''\n-            while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):\n-                text += filedata[j]['data']\n-                del filedata[j]['data']\n-                j += 1\n-\n-            if not stat.has_key('depotFile'):\n-                sys.stderr.write(\"p4 print fails with: %s\\n\" % repr(stat))\n-                continue\n-\n-            if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):\n-                text = re.sub(r'(?i)\\$(Id|Header):[^$]*\\$',r'$\\1$', text)\n-            elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):\n-                text = re.sub(r'\\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\\n]*\\$',r'$\\1$', text)\n-\n-            contents[stat['depotFile']] = text\n-\n-        for f in filesForCommit:\n-            path = f['path']\n-            if contents.has_key(path):\n-                f['data'] = contents[path]\n-\n-        return filesForCommit\n-\n     # output one file from the P4 stream\n     # - helper for streamP4Files\n \n-- \n1.6.2.5\n"},{"id":"118722","messageId":"20090725142416.GC3841@arf.padd.com","threadId":"20085","inReplyTo":"20090725142329.GA3841@arf.padd.com","subject":"[PATCH 2/5] git-p4 stream: do not pass branchPrefixes so much","fromName":"Pete Wyckoff","fromEmail":"pw@padd.com","sentAt":"2009-07-25T14:24:16Z","receivedAt":"2009-07-25T14:24:16Z","isPatch":true,"sender":{"key":"pw@padd.com","avatar":null},"body":"Just set it in self when commit starts and use it everywhere.\nThis makes the code a bit cleaner, but no functional change\n\nSigned-off-by: Pete Wyckoff <pw@padd.com>\n---\n contrib/fast-import/git-p4 |   19 ++++++++++---------\n 1 files changed, 10 insertions(+), 9 deletions(-)\n\ndiff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4\nindex 70fa403..3a52254 100755\n--- a/contrib/fast-import/git-p4\n+++ b/contrib/fast-import/git-p4\n@@ -953,11 +953,11 @@ class P4Sync(Command):\n     # output one file from the P4 stream\n     # - helper for streamP4Files\n \n-    def streamOneP4File(self, file, contents, branchPrefixes):\n+    def streamOneP4File(self, file, contents):\n         if verbose:\n             sys.stderr.write(\"%s\\n\" %  file[\"depotFile\"])\n \n-        relPath = self.stripRepoPath(file['depotFile'], branchPrefixes)\n+        relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)\n \n         mode = \"644\"\n         if isP4Exec(file[\"type\"]):\n@@ -993,16 +993,16 @@ class P4Sync(Command):\n             self.gitStream.write(d)\n         self.gitStream.write(\"\\n\")\n \n-    def streamOneP4Deletion(self, file, branchPrefixes):\n+    def streamOneP4Deletion(self, file):\n         if verbose:\n             sys.stderr.write(\"delete %s\\n\" %  file[\"path\"])\n \n-        relPath = self.stripRepoPath(file['path'], branchPrefixes)\n+        relPath = self.stripRepoPath(file['path'], self.branchPrefixes)\n \n         self.gitStream.write(\"D %s\\n\" % relPath)\n \n     # Stream directly from \"p4 files\" into \"git fast-import\"\n-    def streamP4Files(self, files, branchPrefixes):\n+    def streamP4Files(self, files):\n         filesForCommit = []\n         filesToRead = []\n         filesToDelete = []\n@@ -1026,7 +1026,7 @@ class P4Sync(Command):\n \n         # deleted files...\n         for f in filesToDelete:\n-            self.streamOneP4Deletion(f, branchPrefixes)\n+            self.streamOneP4Deletion(f)\n \n         if len(filesToRead) > 0:\n             stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode='w+b')\n@@ -1058,7 +1058,7 @@ class P4Sync(Command):\n                             continue\n \n \n-                        self.streamOneP4File(file,contents,branchPrefixes)\n+                        self.streamOneP4File(file, contents)\n                         file = {}\n                         contents = []\n                         have_file_info = False\n@@ -1078,7 +1078,7 @@ class P4Sync(Command):\n             # do the last chunk\n \n             if file.has_key('depotFile'):\n-                self.streamOneP4File(file,contents,branchPrefixes)\n+                self.streamOneP4File(file,contents)\n \n             exitCode = p4.wait()\n             if exitCode != 0:\n@@ -1088,6 +1088,7 @@ class P4Sync(Command):\n     def commit(self, details, files, branch, branchPrefixes, parent = \"\"):\n         epoch = details[\"time\"]\n         author = details[\"user\"]\n+\tself.branchPrefixes = branchPrefixes\n \n         if self.verbose:\n             print \"commit into %s\" % branch\n@@ -1127,7 +1128,7 @@ class P4Sync(Command):\n                 print \"parent %s\" % parent\n             self.gitStream.write(\"from %s\\n\" % parent)\n \n-        self.streamP4Files(new_files,branchPrefixes)\n+        self.streamP4Files(new_files)\n         self.gitStream.write(\"\\n\")\n \n         change = int(details[\"change\"])\n-- \n1.6.2.5\n"},{"id":"118725","messageId":"20090725142434.GD3841@arf.padd.com","threadId":"20085","inReplyTo":"20090725142329.GA3841@arf.padd.com","subject":"[PATCH 3/5] git-p4 stream: show relative path in debug messages","fromName":"Pete Wyckoff","fromEmail":"pw@padd.com","sentAt":"2009-07-25T14:24:34Z","receivedAt":"2009-07-25T14:24:34Z","isPatch":true,"sender":{"key":"pw@padd.com","avatar":null},"body":"Not repo path, but the one where git will put the file.  They're\nshorter and allows verifying branchPrefixes too.\n\nSigned-off-by: Pete Wyckoff <pw@padd.com>\n---\n contrib/fast-import/git-p4 |   11 ++++-------\n 1 files changed, 4 insertions(+), 7 deletions(-)\n\ndiff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4\nindex 3a52254..3ab016b 100755\n--- a/contrib/fast-import/git-p4\n+++ b/contrib/fast-import/git-p4\n@@ -954,10 +954,9 @@ class P4Sync(Command):\n     # - helper for streamP4Files\n \n     def streamOneP4File(self, file, contents):\n-        if verbose:\n-            sys.stderr.write(\"%s\\n\" %  file[\"depotFile\"])\n-\n         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)\n+        if verbose:\n+            sys.stderr.write(\"%s\\n\" % relPath)\n \n         mode = \"644\"\n         if isP4Exec(file[\"type\"]):\n@@ -994,11 +993,9 @@ class P4Sync(Command):\n         self.gitStream.write(\"\\n\")\n \n     def streamOneP4Deletion(self, file):\n-        if verbose:\n-            sys.stderr.write(\"delete %s\\n\" %  file[\"path\"])\n-\n         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)\n-\n+        if verbose:\n+            sys.stderr.write(\"delete %s\\n\" % relPath)\n         self.gitStream.write(\"D %s\\n\" % relPath)\n \n     # Stream directly from \"p4 files\" into \"git fast-import\"\n-- \n1.6.2.5\n"},{"id":"118724","messageId":"20090725142451.GE3841@arf.padd.com","threadId":"20085","inReplyTo":"20090725142329.GA3841@arf.padd.com","subject":"[PATCH 4/5] git-p4 stream: check apple file type","fromName":"Pete Wyckoff","fromEmail":"pw@padd.com","sentAt":"2009-07-25T14:24:51Z","receivedAt":"2009-07-25T14:24:51Z","isPatch":true,"sender":{"key":"pw@padd.com","avatar":null},"body":"Move this check into the function so both callers do it.\nAnd always reset the file contents in the main loop, as\nwe just ignored this apple file.\n\nSigned-off-by: Pete Wyckoff <pw@padd.com>\n---\n contrib/fast-import/git-p4 |   11 +++++------\n 1 files changed, 5 insertions(+), 6 deletions(-)\n\ndiff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4\nindex 3ab016b..af66026 100755\n--- a/contrib/fast-import/git-p4\n+++ b/contrib/fast-import/git-p4\n@@ -954,6 +954,11 @@ class P4Sync(Command):\n     # - helper for streamP4Files\n \n     def streamOneP4File(self, file, contents):\n+\tif file[\"type\"] == \"apple\":\n+\t    print \"\\nfile %s is a strange apple file that forks. Ignoring\" % \\\n+\t    \t  file['depotFile']\n+\t    return\n+\n         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)\n         if verbose:\n             sys.stderr.write(\"%s\\n\" % relPath)\n@@ -1049,12 +1054,6 @@ class P4Sync(Command):\n \n                     if marshalled.has_key('depotFile') and have_file_info:\n                         # start of a new file - output the old one first\n-\n-                        if file[\"type\"] == \"apple\":\n-                            print \"\\nfile %s is a strange apple file that forks. Ignoring\" % file['path']\n-                            continue\n-\n-\n                         self.streamOneP4File(file, contents)\n                         file = {}\n                         contents = []\n-- \n1.6.2.5\n"},{"id":"118720","messageId":"20090725142509.GF3841@arf.padd.com","threadId":"20085","inReplyTo":"20090725142329.GA3841@arf.padd.com","subject":"[PATCH 5/5] git-p4 stream: use existing p4CmdList with callback","fromName":"Pete Wyckoff","fromEmail":"pw@padd.com","sentAt":"2009-07-25T14:25:09Z","receivedAt":"2009-07-25T14:25:09Z","isPatch":true,"sender":{"key":"pw@padd.com","avatar":null},"body":"Add a callback argrument to iterate over returned contents\nrather than replicate the entire function just to do that.\n\nSigned-off-by: Pete Wyckoff <pw@padd.com>\n---\n contrib/fast-import/git-p4 |   88 +++++++++++++++++++-------------------------\n 1 files changed, 38 insertions(+), 50 deletions(-)\n\ndiff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4\nindex af66026..eece984 100755\n--- a/contrib/fast-import/git-p4\n+++ b/contrib/fast-import/git-p4\n@@ -201,7 +201,7 @@ def isModeExec(mode):\n def isModeExecChanged(src_mode, dst_mode):\n     return isModeExec(src_mode) != isModeExec(dst_mode)\n \n-def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):\n+def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):\n     cmd = p4_build_cmd(\"-G %s\" % (cmd))\n     if verbose:\n         sys.stderr.write(\"Opening pipe: %s\\n\" % cmd)\n@@ -224,7 +224,10 @@ def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):\n     try:\n         while True:\n             entry = marshal.load(p4.stdout)\n-            result.append(entry)\n+\t    if cb is not None:\n+\t\tcb(entry)\n+\t    else:\n+\t\tresult.append(entry)\n     except EOFError:\n         pass\n     exitCode = p4.wait()\n@@ -1003,6 +1006,26 @@ class P4Sync(Command):\n             sys.stderr.write(\"delete %s\\n\" % relPath)\n         self.gitStream.write(\"D %s\\n\" % relPath)\n \n+    # handle another chunk of streaming data\n+    def streamP4FilesCb(self, marshalled):\n+\n+\tif marshalled.has_key('depotFile') and self.stream_have_file_info:\n+\t    # start of a new file - output the old one first\n+\t    self.streamOneP4File(self.stream_file, self.stream_contents)\n+\t    self.stream_file = {}\n+\t    self.stream_contents = []\n+\t    self.stream_have_file_info = False\n+\n+\t# pick up the new file information... for the\n+\t# 'data' field we need to append to our array\n+\tfor k in marshalled.keys():\n+\t    if k == 'data':\n+\t\tself.stream_contents.append(marshalled['data'])\n+\t    else:\n+\t\tself.stream_file[k] = marshalled[k]\n+\n+\tself.stream_have_file_info = True\n+\n     # Stream directly from \"p4 files\" into \"git fast-import\"\n     def streamP4Files(self, files):\n         filesForCommit = []\n@@ -1024,62 +1047,27 @@ class P4Sync(Command):\n                 else:\n                     filesToDelete.append(f)\n \n-        filedata = []\n-\n         # deleted files...\n         for f in filesToDelete:\n             self.streamOneP4Deletion(f)\n \n         if len(filesToRead) > 0:\n-            stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode='w+b')\n-            stdin_file.write('\\n'.join(['%s#%s' % (f['path'], f['rev'])\n-                                                  for f in filesToRead]))\n-            stdin_file.flush()\n-            stdin_file.seek(0)\n-            try:\n-                p4 = subprocess.Popen('p4 -G -x - print',\n-                                        shell=True,\n-                                        stdin=stdin_file,\n-                                        stdout=subprocess.PIPE);\n-            except OSError,e:\n-                print >> sys.stderr, \"p4 print failed:\", e\n+            self.stream_file = {}\n+            self.stream_contents = []\n+            self.stream_have_file_info = False\n \n-            file = {}\n-            contents = []\n-            have_file_info = False\n+\t    # curry self argument\n+\t    def streamP4FilesCbSelf(entry):\n+\t\tself.streamP4FilesCb(entry)\n \n-            try:\n-                while True:\n-                    marshalled = marshal.load(p4.stdout)\n-\n-                    if marshalled.has_key('depotFile') and have_file_info:\n-                        # start of a new file - output the old one first\n-                        self.streamOneP4File(file, contents)\n-                        file = {}\n-                        contents = []\n-                        have_file_info = False\n-\n-                    # pick up the new file information... for the\n-                    # 'data' field we need to append to our array\n-                    for k in marshalled.keys():\n-                        if k == 'data':\n-                            contents.append(marshalled['data'])\n-                        else:\n-                            file[k] = marshalled[k]\n-\n-                    have_file_info = True\n-            except EOFError:\n-                pass\n+\t    p4CmdList(\"-x - print\",\n+\t\t'\\n'.join(['%s#%s' % (f['path'], f['rev'])\n+                                                  for f in filesToRead]),\n+\t        cb=streamP4FilesCbSelf)\n \n             # do the last chunk\n-\n-            if file.has_key('depotFile'):\n-                self.streamOneP4File(file,contents)\n-\n-            exitCode = p4.wait()\n-            if exitCode != 0:\n-                sys.stderr.write(\"p4 subshell failed getting file data\\n\")\n-                sys.exit(1)\n+            if self.stream_file.has_key('depotFile'):\n+                self.streamOneP4File(self.stream_file, self.stream_contents)\n \n     def commit(self, details, files, branch, branchPrefixes, parent = \"\"):\n         epoch = details[\"time\"]\n-- \n1.6.2.5\n"}]}