{"thread":{"id":"20083","subject":"[PATCH/RFC] git-p4: stream from perforce to speed up clones","startedAt":"2009-07-10T20:10:44Z","lastAt":"2009-07-10T20:10:44Z","messageCount":1,"participants":["Luke Diamand"],"isPatch":true,"patchVersion":1,"patchTotal":null},"messages":[{"id":"117796","messageId":"200907102110.44973.luke@diamand.org","threadId":"20083","inReplyTo":null,"subject":"[PATCH/RFC] git-p4: stream from perforce to speed up clones","fromName":"Luke Diamand","fromEmail":"luke@diamand.org","sentAt":"2009-07-10T20:10:44Z","receivedAt":"2009-07-10T20:10:44Z","isPatch":true,"sender":{"key":"luke@diamand.org","avatar":"https://avatars.githubusercontent.com/u/5330967?v=4"},"body":"Change commit() to stream data from Perforce and into fast-import\nrather than reading into memory first, and then writing out. This\nhugely reduces the memory requirements when cloning non-incrementally.\n\nSigned-off-by: Luke Diamand <luke@diamand.org>\n---\nI've modified git-p4 so that it streams/pipes data into fast-import rather\nthat reading everything into memory first. The old scheme meant that\nfor a large repository (mine is around 2G) my PC just grinds to a\nhalt and never actually finishes. With this change it takes around ten\nminutes.\n\nObviously there's scope here for breaking git-p4 quite badly. I have a\nsmall test harness that covers regular and incremental imports, file\ndeletion and keyword expansion, but perhaps there are other aspects\nI need to look at as well?\n\n contrib/fast-import/git-p4 |  164 ++++++++++++++++++++++++++++++++++++--------\n 1 files changed, 136 insertions(+), 28 deletions(-)\n\ndiff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4\nindex 342529d..f415ad0 100755\n--- a/contrib/fast-import/git-p4\n+++ b/contrib/fast-import/git-p4\n@@ -1008,6 +1008,141 @@ class P4Sync(Command):\n \n         return filesForCommit\n \n+    # output one file from the P4 stream\n+    # - helper for streamP4Files\n+\n+    def streamOneP4File(self, file, contents, branchPrefixes):\n+        if verbose:\n+            sys.stderr.write(\"%s\\n\" %  file[\"depotFile\"])\n+\n+        relPath = self.stripRepoPath(file['depotFile'], branchPrefixes)\n+\n+        mode = \"644\"\n+        if isP4Exec(file[\"type\"]):\n+            mode = \"755\"\n+        elif file[\"type\"] == \"symlink\":\n+            mode = \"120000\"\n+            # p4 print on a symlink contains \"target\\n\", so strip it off\n+            last = contents.pop()\n+            last = last[:-1]\n+            contents.append(last)\n+\n+        if self.isWindows and file[\"type\"].endswith(\"text\"):\n+            mangled = []\n+            for data in contents:\n+                data = data.replace(\"\\r\\n\", \"\\n\")\n+                mangled.append(data)\n+            contents = mangled\n+\n+        if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):\n+            contents = map(lambda text: re.sub(r'(?i)\\$(Id|Header):[^$]*\\$',r'$\\1$', text), contents)\n+        elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):\n+            contents = map(lambda text: re.sub(r'\\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\\n]*\\$',r'$\\1$', text), contents)\n+\n+        self.gitStream.write(\"M %s inline %s\\n\" % (mode, relPath))\n+\n+        # total length...\n+        length = 0\n+        for d in contents:\n+            length = length + len(d)\n+\n+        self.gitStream.write(\"data %d\\n\" % length)\n+        for d in contents:\n+            self.gitStream.write(d)\n+        self.gitStream.write(\"\\n\")\n+\n+    def streamOneP4Deletion(self, file, branchPrefixes):\n+        if verbose:\n+            sys.stderr.write(\"delete %s\\n\" %  file[\"path\"])\n+\n+        relPath = self.stripRepoPath(file['path'], branchPrefixes)\n+\n+        self.gitStream.write(\"D %s\\n\" % relPath)\n+\n+    # Stream directly from \"p4 files\" into \"git fast-import\"\n+    def streamP4Files(self, files, branchPrefixes):\n+        filesForCommit = []\n+        filesToRead = []\n+        filesToDelete = []\n+\n+        for f in files:\n+            includeFile = True\n+            for val in self.clientSpecDirs:\n+                if f['path'].startswith(val[0]):\n+                    if val[1] <= 0:\n+                        includeFile = False\n+                    break\n+\n+            if includeFile:\n+                filesForCommit.append(f)\n+                if f['action'] not in ('delete', 'purge'):\n+                    filesToRead.append(f)\n+                else:\n+                    filesToDelete.append(f)\n+\n+        filedata = []\n+\n+        # deleted files...\n+        for f in filesToDelete:\n+            self.streamOneP4Deletion(f, branchPrefixes)\n+\n+        if len(filesToRead) > 0:\n+            stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode='w+b')\n+            stdin_file.write('\\n'.join(['%s#%s' % (f['path'], f['rev'])\n+                                                  for f in filesToRead]))\n+            stdin_file.flush()\n+            stdin_file.seek(0)\n+            try:\n+                p4 = subprocess.Popen('p4 -G -x - print',\n+                                        shell=True,\n+                                        stdin=stdin_file,\n+                                        stdout=subprocess.PIPE);\n+            except OSError,e:\n+                print >> sys.stderr, \"p4 print failed:\", e\n+\n+            file = {}\n+            contents = []\n+            have_file_info = False\n+\n+            try:\n+                while True:\n+                    marshalled = marshal.load(p4.stdout)\n+\n+                    if marshalled.has_key('depotFile') and have_file_info:\n+                        # start of a new file - output the old one first\n+\n+                        if file[\"type\"] == \"apple\":\n+                            print \"\\nfile %s is a strange apple file that forks. Ignoring\" % file['path']\n+                            continue\n+\n+\n+                        self.streamOneP4File(file,contents,branchPrefixes)\n+                        file = {}\n+                        contents = []\n+                        have_file_info = False\n+\n+                    # pick up the new file information... for the\n+                    # 'data' field we need to append to our array\n+                    for k in marshalled.keys():\n+                        if k == 'data':\n+                            contents.append(marshalled['data'])\n+                        else:\n+                            file[k] = marshalled[k]\n+\n+                    have_file_info = True\n+            except EOFError:\n+                pass\n+\n+            # do the last chunk\n+\n+            if file.has_key('depotFile'):\n+                self.streamOneP4File(file,contents,branchPrefixes)\n+\n+            exitCode = p4.wait()\n+            if exitCode != 0:\n+                sys.stderr.write(\"p4 subshell failed getting file data\\n\")\n+                sys.exit(1)\n+\n     def commit(self, details, files, branch, branchPrefixes, parent = \"\"):\n         epoch = details[\"time\"]\n         author = details[\"user\"]\n@@ -1023,7 +1158,6 @@ class P4Sync(Command):\n                 new_files.append (f)\n             else:\n                 sys.stderr.write(\"Ignoring file outside of prefix: %s\\n\" % path)\n-        files = self.readP4Files(new_files)\n \n         self.gitStream.write(\"commit %s\\n\" % branch)\n #        gitStream.write(\"mark :%s\\n\" % details[\"change\"])\n@@ -1051,33 +1185,7 @@ class P4Sync(Command):\n                 print \"parent %s\" % parent\n             self.gitStream.write(\"from %s\\n\" % parent)\n \n-        for file in files:\n-            if file[\"type\"] == \"apple\":\n-                print \"\\nfile %s is a strange apple file that forks. Ignoring!\" % file['path']\n-                continue\n-\n-            relPath = self.stripRepoPath(file['path'], branchPrefixes)\n-            if file[\"action\"] in (\"delete\", \"purge\"):\n-                self.gitStream.write(\"D %s\\n\" % relPath)\n-            else:\n-                data = file['data']\n-\n-                mode = \"644\"\n-                if isP4Exec(file[\"type\"]):\n-                    mode = \"755\"\n-                elif file[\"type\"] == \"symlink\":\n-                    mode = \"120000\"\n-                    # p4 print on a symlink contains \"target\\n\", so strip it off\n-                    data = data[:-1]\n-\n-                if self.isWindows and file[\"type\"].endswith(\"text\"):\n-                    data = data.replace(\"\\r\\n\", \"\\n\")\n-\n-                self.gitStream.write(\"M %s inline %s\\n\" % (mode, relPath))\n-                self.gitStream.write(\"data %s\\n\" % len(data))\n-                self.gitStream.write(data)\n-                self.gitStream.write(\"\\n\")\n-\n+        self.streamP4Files(new_files,branchPrefixes)\n         self.gitStream.write(\"\\n\")\n \n         change = int(details[\"change\"])\n-- \n1.6.3.GIT\n"}]}