{"thread":{"id":"20287","subject":"[PATCH v3] git-p4: stream from perforce to speed up clones","startedAt":"2009-07-29T23:13:46Z","lastAt":"2009-07-29T23:13:46Z","messageCount":1,"participants":["Luke Diamand"],"isPatch":true,"patchVersion":3,"patchTotal":null},"messages":[{"id":"119119","messageId":"1248909226-17526-1-git-send-email-luke@diamand.org","threadId":"20287","inReplyTo":null,"subject":"[PATCH v3] git-p4: stream from perforce to speed up clones","fromName":"Luke Diamand","fromEmail":"luke@diamand.org","sentAt":"2009-07-29T23:13:46Z","receivedAt":"2009-07-29T23:13:46Z","isPatch":true,"sender":{"key":"luke@diamand.org","avatar":"https://avatars.githubusercontent.com/u/5330967?v=4"},"body":"Change commit() to stream data from Perforce and into fast-import\nrather than reading into memory first, and then writing out. This\nhugely reduces the memory requirements when cloning non-incrementally.\n\nSigned-off-by: Luke Diamand <luke@diamand.org>\nSigned-off-by: Pete Wyckoff <pw@padd.com>\n---\nI've modified git-p4 so that it streams/pipes data into fast-import rather\nthat reading everything into memory first. The old scheme meant that\nfor a large repository (mine is around 2G) my PC just grinds to a\nhalt and never actually finishes. With this change it takes around ten\nminutes.\n\nPete Wyckoff kindly submitted some patches on top of this which tidied it\nup and fixed a few bugs; this patch merges my original patch with those\nchanges.\n\nSimon - could you review this change please?\n\nThanks,\nLuke\n\n contrib/fast-import/git-p4 |  169 ++++++++++++++++++++++++++-----------------\n 1 files changed, 102 insertions(+), 67 deletions(-)\n\ndiff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4\nindex 342529d..38438f3 100755\n--- a/contrib/fast-import/git-p4\n+++ b/contrib/fast-import/git-p4\n@@ -201,7 +201,7 @@ def isModeExec(mode):\n def isModeExecChanged(src_mode, dst_mode):\n     return isModeExec(src_mode) != isModeExec(dst_mode)\n \n-def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):\n+def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):\n     cmd = p4_build_cmd(\"-G %s\" % (cmd))\n     if verbose:\n         sys.stderr.write(\"Opening pipe: %s\\n\" % cmd)\n@@ -224,7 +224,10 @@ def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):\n     try:\n         while True:\n             entry = marshal.load(p4.stdout)\n-            result.append(entry)\n+\t    if cb is not None:\n+\t\tcb(entry)\n+\t    else:\n+\t\tresult.append(entry)\n     except EOFError:\n         pass\n     exitCode = p4.wait()\n@@ -950,10 +953,84 @@ class P4Sync(Command):\n \n         return branches\n \n-    ## Should move this out, doesn't use SELF.\n-    def readP4Files(self, files):\n+    # output one file from the P4 stream\n+    # - helper for streamP4Files\n+\n+    def streamOneP4File(self, file, contents):\n+\tif file[\"type\"] == \"apple\":\n+\t    print \"\\nfile %s is a strange apple file that forks. Ignoring\" % \\\n+\t\tfile['depotFile']\n+\t    return\n+\n+        relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)\n+        if verbose:\n+            sys.stderr.write(\"%s\\n\" % relPath)\n+\n+        mode = \"644\"\n+        if isP4Exec(file[\"type\"]):\n+            mode = \"755\"\n+        elif file[\"type\"] == \"symlink\":\n+            mode = \"120000\"\n+            # p4 print on a symlink contains \"target\\n\", so strip it off\n+            last = contents.pop()\n+            last = last[:-1]\n+            contents.append(last)\n+\n+        if self.isWindows and file[\"type\"].endswith(\"text\"):\n+            mangled = []\n+            for data in contents:\n+                data = data.replace(\"\\r\\n\", \"\\n\")\n+                mangled.append(data)\n+            contents = mangled\n+\n+        if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):\n+            contents = map(lambda text: re.sub(r'(?i)\\$(Id|Header):[^$]*\\$',r'$\\1$', text), contents)\n+        elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):\n+            contents = map(lambda text: re.sub(r'\\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\\n]*\\$',r'$\\1$', text), contents)\n+\n+        self.gitStream.write(\"M %s inline %s\\n\" % (mode, relPath))\n+\n+        # total length...\n+        length = 0\n+        for d in contents:\n+            length = length + len(d)\n+\n+        self.gitStream.write(\"data %d\\n\" % length)\n+        for d in contents:\n+            self.gitStream.write(d)\n+        self.gitStream.write(\"\\n\")\n+\n+    def streamOneP4Deletion(self, file):\n+        relPath = self.stripRepoPath(file['path'], self.branchPrefixes)\n+        if verbose:\n+            sys.stderr.write(\"delete %s\\n\" % relPath)\n+        self.gitStream.write(\"D %s\\n\" % relPath)\n+\n+    # handle another chunk of streaming data\n+    def streamP4FilesCb(self, marshalled):\n+\n+\tif marshalled.has_key('depotFile') and self.stream_have_file_info:\n+\t    # start of a new file - output the old one first\n+\t    self.streamOneP4File(self.stream_file, self.stream_contents)\n+\t    self.stream_file = {}\n+\t    self.stream_contents = []\n+\t    self.stream_have_file_info = False\n+\n+\t# pick up the new file information... for the\n+\t# 'data' field we need to append to our array\n+\tfor k in marshalled.keys():\n+\t    if k == 'data':\n+\t\tself.stream_contents.append(marshalled['data'])\n+\t    else:\n+\t\tself.stream_file[k] = marshalled[k]\n+\n+\tself.stream_have_file_info = True\n+\n+    # Stream directly from \"p4 files\" into \"git fast-import\"\n+    def streamP4Files(self, files):\n         filesForCommit = []\n         filesToRead = []\n+        filesToDelete = []\n \n         for f in files:\n             includeFile = True\n@@ -967,50 +1044,35 @@ class P4Sync(Command):\n                 filesForCommit.append(f)\n                 if f['action'] not in ('delete', 'purge'):\n                     filesToRead.append(f)\n+                else:\n+                    filesToDelete.append(f)\n \n-        filedata = []\n-        if len(filesToRead) > 0:\n-            filedata = p4CmdList('-x - print',\n-                                 stdin='\\n'.join(['%s#%s' % (f['path'], f['rev'])\n-                                                  for f in filesToRead]),\n-                                 stdin_mode='w+')\n-\n-            if \"p4ExitCode\" in filedata[0]:\n-                die(\"Problems executing p4. Error: [%d].\"\n-                    % (filedata[0]['p4ExitCode']));\n-\n-        j = 0;\n-        contents = {}\n-        while j < len(filedata):\n-            stat = filedata[j]\n-            j += 1\n-            text = ''\n-            while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):\n-                text += filedata[j]['data']\n-                del filedata[j]['data']\n-                j += 1\n-\n-            if not stat.has_key('depotFile'):\n-                sys.stderr.write(\"p4 print fails with: %s\\n\" % repr(stat))\n-                continue\n+        # deleted files...\n+        for f in filesToDelete:\n+            self.streamOneP4Deletion(f)\n \n-            if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):\n-                text = re.sub(r'(?i)\\$(Id|Header):[^$]*\\$',r'$\\1$', text)\n-            elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):\n-                text = re.sub(r'\\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\\n]*\\$',r'$\\1$', text)\n+        if len(filesToRead) > 0:\n+            self.stream_file = {}\n+            self.stream_contents = []\n+            self.stream_have_file_info = False\n \n-            contents[stat['depotFile']] = text\n+\t    # curry self argument\n+\t    def streamP4FilesCbSelf(entry):\n+\t\tself.streamP4FilesCb(entry)\n \n-        for f in filesForCommit:\n-            path = f['path']\n-            if contents.has_key(path):\n-                f['data'] = contents[path]\n+\t    p4CmdList(\"-x - print\",\n+\t\t'\\n'.join(['%s#%s' % (f['path'], f['rev'])\n+                                                  for f in filesToRead]),\n+\t        cb=streamP4FilesCbSelf)\n \n-        return filesForCommit\n+            # do the last chunk\n+            if self.stream_file.has_key('depotFile'):\n+                self.streamOneP4File(self.stream_file, self.stream_contents)\n \n     def commit(self, details, files, branch, branchPrefixes, parent = \"\"):\n         epoch = details[\"time\"]\n         author = details[\"user\"]\n+\tself.branchPrefixes = branchPrefixes\n \n         if self.verbose:\n             print \"commit into %s\" % branch\n@@ -1023,7 +1085,6 @@ class P4Sync(Command):\n                 new_files.append (f)\n             else:\n                 sys.stderr.write(\"Ignoring file outside of prefix: %s\\n\" % path)\n-        files = self.readP4Files(new_files)\n \n         self.gitStream.write(\"commit %s\\n\" % branch)\n #        gitStream.write(\"mark :%s\\n\" % details[\"change\"])\n@@ -1051,33 +1112,7 @@ class P4Sync(Command):\n                 print \"parent %s\" % parent\n             self.gitStream.write(\"from %s\\n\" % parent)\n \n-        for file in files:\n-            if file[\"type\"] == \"apple\":\n-                print \"\\nfile %s is a strange apple file that forks. Ignoring!\" % file['path']\n-                continue\n-\n-            relPath = self.stripRepoPath(file['path'], branchPrefixes)\n-            if file[\"action\"] in (\"delete\", \"purge\"):\n-                self.gitStream.write(\"D %s\\n\" % relPath)\n-            else:\n-                data = file['data']\n-\n-                mode = \"644\"\n-                if isP4Exec(file[\"type\"]):\n-                    mode = \"755\"\n-                elif file[\"type\"] == \"symlink\":\n-                    mode = \"120000\"\n-                    # p4 print on a symlink contains \"target\\n\", so strip it off\n-                    data = data[:-1]\n-\n-                if self.isWindows and file[\"type\"].endswith(\"text\"):\n-                    data = data.replace(\"\\r\\n\", \"\\n\")\n-\n-                self.gitStream.write(\"M %s inline %s\\n\" % (mode, relPath))\n-                self.gitStream.write(\"data %s\\n\" % len(data))\n-                self.gitStream.write(data)\n-                self.gitStream.write(\"\\n\")\n-\n+        self.streamP4Files(new_files)\n         self.gitStream.write(\"\\n\")\n \n         change = int(details[\"change\"])\n-- \n1.6.3.GIT\n"}]}