{"thread":{"id":"40143","subject":"[PATCH v3] git-p4: fix faulty paths for case insensitive systems","startedAt":"2015-08-20T21:43:54Z","lastAt":"2015-08-20T21:43:54Z","messageCount":1,"participants":["larsxschneider@gmail.com"],"isPatch":true,"patchVersion":3,"patchTotal":null},"messages":[{"id":"268389","messageId":"1440107034-90735-1-git-send-email-larsxschneider@gmail.com","threadId":"40143","inReplyTo":null,"subject":"[PATCH v3] git-p4: fix faulty paths for case insensitive systems","fromName":"","fromEmail":"larsxschneider@gmail.com","sentAt":"2015-08-20T21:43:54Z","receivedAt":"2015-08-20T21:43:54Z","isPatch":true,"sender":{"key":"larsxschneider@gmail.com","avatar":"https://avatars.githubusercontent.com/u/477434?v=4"},"body":"From: Lars Schneider <larsxschneider@gmail.com>\n\nPROBLEM:\nWe run P4 servers on Linux and P4 clients on Windows. For an unknown\nreason the file path for a number of files in P4 does not match the\ndirectory path with respect to case sensitivity.\n\nE.g. `p4 files` might return\n//depot/path/to/file1\n//depot/PATH/to/file2\n\nIf you use P4/P4V then these files end up in the same directory, e.g.\n//depot/path/to/file1\n//depot/path/to/file2\n\nIf you use git-p4 then all files not matching the correct file path\n(e.g. `file2`) will be ignored.\n\nSOLUTION:\nIdentify path names that are different with respect to case sensitivity.\nIf there are any then run `p4 dirs` to build up a dictionary\ncontaining the \"correct\" cases for each path. It looks like P4\ninterprets \"correct\" here as the existing path of the first file in a\ndirectory. The path dictionary is used later on to fix all paths.\n\nThis is only applied if the parameter \"--ignore-path-case\" is passed to\nthe git-p4 clone command.\n\nSigned-off-by: Lars Schneider <larsxschneider@gmail.com>\n---\n git-p4.py                         |  83 ++++++++++++++++++++++++++++--\n t/t9821-git-p4-path-variations.sh | 103 ++++++++++++++++++++++++++++++++++++++\n 2 files changed, 182 insertions(+), 4 deletions(-)\n create mode 100755 t/t9821-git-p4-path-variations.sh\n\ndiff --git a/git-p4.py b/git-p4.py\nindex 073f87b..8a2a998 100755\n--- a/git-p4.py\n+++ b/git-p4.py\n@@ -1931,7 +1931,7 @@ class View(object):\n                 (self.client_prefix, clientFile))\n         return clientFile[len(self.client_prefix):]\n\n-    def update_client_spec_path_cache(self, files):\n+    def update_client_spec_path_cache(self, files, fixPathCase = None):\n         \"\"\" Caching file paths by \"p4 where\" batch query \"\"\"\n\n         # List depot file paths exclude that already cached\n@@ -1950,6 +1950,8 @@ class View(object):\n             if \"unmap\" in res:\n                 # it will list all of them, but only one not unmap-ped\n                 continue\n+            if fixPathCase:\n+                res['depotFile'] = fixPathCase(res['depotFile'])\n             self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res[\"clientFile\"])\n\n         # not found files or unmap files set to \"\"\n@@ -1987,6 +1989,7 @@ class P4Sync(Command, P4UserMap):\n                                      help=\"Maximum number of changes to import\"),\n                 optparse.make_option(\"--changes-block-size\", dest=\"changes_block_size\", type=\"int\",\n                                      help=\"Internal block size to use when iteratively calling p4 changes\"),\n+                optparse.make_option(\"--ignore-path-case\", dest=\"ignorePathCase\", action=\"store_true\"),\n                 optparse.make_option(\"--keep-path\", dest=\"keepRepoPath\", action='store_true',\n                                      help=\"Keep entire BRANCH/DIR/SUBDIR prefix during import\"),\n                 optparse.make_option(\"--use-client-spec\", dest=\"useClientSpec\", action='store_true',\n@@ -2017,6 +2020,7 @@ class P4Sync(Command, P4UserMap):\n         self.maxChanges = \"\"\n         self.changes_block_size = None\n         self.keepRepoPath = False\n+        self.ignorePathCase = False\n         self.depotPaths = None\n         self.p4BranchesInGit = []\n         self.cloneExclude = []\n@@ -2049,7 +2053,8 @@ class P4Sync(Command, P4UserMap):\n         files = []\n         fnum = 0\n         while commit.has_key(\"depotFile%s\" % fnum):\n-            path =  commit[\"depotFile%s\" % fnum]\n+            path = commit[\"depotFile%s\" % fnum]\n+            path = self.fixPathCase(path)\n\n             if [p for p in self.cloneExclude\n                 if p4PathStartsWith(path, p)]:\n@@ -2113,7 +2118,9 @@ class P4Sync(Command, P4UserMap):\n         branches = {}\n         fnum = 0\n         while commit.has_key(\"depotFile%s\" % fnum):\n-            path =  commit[\"depotFile%s\" % fnum]\n+            path = commit[\"depotFile%s\" % fnum]\n+            path = self.fixPathCase(path)\n+\n             found = [p for p in self.depotPaths\n                      if p4PathStartsWith(path, p)]\n             if not found:\n@@ -2240,6 +2247,10 @@ class P4Sync(Command, P4UserMap):\n             if marshalled[\"code\"] == \"error\":\n                 if \"data\" in marshalled:\n                     err = marshalled[\"data\"].rstrip()\n+\n+        if \"depotFile\" in marshalled:\n+            marshalled['depotFile'] = self.fixPathCase(marshalled['depotFile'])\n+\n         if err:\n             f = None\n             if self.stream_have_file_info:\n@@ -2314,6 +2325,7 @@ class P4Sync(Command, P4UserMap):\n\n             # do the last chunk\n             if self.stream_file.has_key('depotFile'):\n+                self.stream_file['depotFile'] = self.fixPathCase(self.stream_file['depotFile'])\n                 self.streamOneP4File(self.stream_file, self.stream_contents)\n\n     def make_email(self, userid):\n@@ -2371,7 +2383,8 @@ class P4Sync(Command, P4UserMap):\n                 sys.stderr.write(\"Ignoring file outside of prefix: %s\\n\" % f['path'])\n\n         if self.clientSpecDirs:\n-            self.clientSpecDirs.update_client_spec_path_cache(files)\n+            self.clientSpecDirs.update_client_spec_path_cache(\n+                files, lambda x: self.fixPathCase(x))\n\n         self.gitStream.write(\"commit %s\\n\" % branch)\n #        gitStream.write(\"mark :%s\\n\" % details[\"change\"])\n@@ -2835,6 +2848,63 @@ class P4Sync(Command, P4UserMap):\n             print \"IO error with git fast-import. Is your git version recent enough?\"\n             print self.gitError.read()\n\n+    def fixPathCase(self, path):\n+        if self.caseCorrectedPaths:\n+            components = path.split('/')\n+            filename = components.pop()\n+            dirname = '/'.join(components).lower() + '/'\n+            if dirname in self.caseCorrectedPaths:\n+                path = self.caseCorrectedPaths[dirname] + filename\n+        return path\n+\n+    def generatePathCaseDict(self, depotPaths):\n+        # Query all files and generate a list of all used paths\n+        # e.g. this files list:\n+        # //depot/path/to/file1\n+        # //depot/PATH/to/file2\n+        #\n+        # result in this path list:\n+        # //depot/\n+        # //depot/PATH/\n+        # //depot/path/\n+        # //depot/PATH/to/\n+        # //depot/path/to/\n+        p4_paths = set()\n+        for p in depotPaths:\n+            for f in p4CmdList([\"files\", p+\"...\"]):\n+                components = f[\"depotFile\"].split('/')[0:-1]\n+                for i in range(3, len(components)+1):\n+                    p4_paths.add('/'.join(components[0:i]) + '/')\n+        p4_paths = sorted(list(p4_paths), key=len)\n+\n+        if len(p4_paths) > len(set([p.lower() for p in p4_paths])):\n+            print \"ATTENTION: File paths with different case variations detected. Fixing may take a while...\"\n+            found_variations = True\n+            while found_variations:\n+                for path in p4_paths:\n+                    found_variations = False\n+                    path_variations = [p for p in p4_paths if p.lower() == path.lower()]\n+\n+                    if len(path_variations) > 1:\n+                        print  \"%i different case variations for path '%s' detected.\" % (len(path_variations), path)\n+                        # If we detect path variations (e.g. //depot/path and //depot/PATH) then we query P4 to list\n+                        # the subdirectories of the parent (e.g //depot/*). P4 will return these subdirectories with\n+                        # the correct case.\n+                        parent_path = '/'.join(path.split('/')[0:-2])\n+                        case_ok_paths = [p[\"dir\"] + '/' for p in p4CmdList([\"dirs\", \"-D\", parent_path + '/*'])]\n+\n+                        # Replace all known paths with the case corrected path from P4 dirs command\n+                        for case_ok_path in case_ok_paths:\n+                            pattern = re.compile(\"^\" + re.escape(case_ok_path), re.IGNORECASE)\n+                            p4_paths = sorted(list(set([pattern.sub(case_ok_path, p) for p in p4_paths])), key=len)\n+\n+                        found_variations = True\n+                        break\n+            return dict((p.lower(), p) for p in p4_paths)\n+        else:\n+            if self.verbose:\n+                print \"All file paths have consistent case\"\n+            return None\n\n     def run(self, args):\n         self.depotPaths = []\n@@ -3006,6 +3076,11 @@ class P4Sync(Command, P4UserMap):\n\n         self.depotPaths = newPaths\n\n+        if self.ignorePathCase:\n+            self.caseCorrectedPaths = self.generatePathCaseDict(self.depotPaths)\n+        else:\n+            self.caseCorrectedPaths = None\n+\n         # --detect-branches may change this for each branch\n         self.branchPrefixes = self.depotPaths\n\ndiff --git a/t/t9821-git-p4-path-variations.sh b/t/t9821-git-p4-path-variations.sh\nnew file mode 100755\nindex 0000000..ec5aa7d\n--- /dev/null\n+++ b/t/t9821-git-p4-path-variations.sh\n@@ -0,0 +1,103 @@\n+#!/bin/sh\n+\n+test_description='Clone repositories with path case variations'\n+\n+. ./lib-git-p4.sh\n+\n+test_expect_success 'start p4d with case folding enabled' '\n+\tstart_p4d -C1\n+'\n+\n+test_expect_success 'Create a repo with path case variations' '\n+\tclient_view \"//depot/... //client/...\" &&\n+\t(\n+\t\tcd \"$cli\" &&\n+\n+\t\tmkdir -p One/two &&\n+\t\t>One/two/File2.txt &&\n+\t\tp4 add One/two/File2.txt &&\n+\t\tp4 submit -d \"Add file2\" &&\n+\t\trm -rf One &&\n+\n+\t\tmkdir -p one/TWO &&\n+\t\t>one/TWO/file1.txt &&\n+\t\tp4 add one/TWO/file1.txt &&\n+\t\tp4 submit -d \"Add file1\" &&\n+\t\trm -rf one &&\n+\n+\t\tmkdir -p one/two &&\n+\t\t>one/two/file3.txt &&\n+\t\tp4 add one/two/file3.txt &&\n+\t\tp4 submit -d \"Add file3\" &&\n+\t\trm -rf one\n+\t)\n+'\n+\n+test_expect_success 'Clone the repo and WITHOUT path fixing' '\n+\tclient_view \"//depot/One/... //client/...\" &&\n+\tgit p4 clone --use-client-spec --destination=\"$git\" //depot/one &&\n+\ttest_when_finished cleanup_git &&\n+\t(\n+\t\tcd \"$git\" &&\n+\t\t# This method is used instead of \"test -f\" to ensure the case is\n+\t\t# checked even if the test is executed on case-insensitive file systems.\n+\t\tcat >expect <<-\\EOF &&\n+\t\t\ttwo/File2.txt\n+\t\tEOF\n+\t\tgit ls-files >actual &&\n+\t\ttest_cmp expect actual\n+\t)\n+'\n+\n+test_expect_success 'Clone the repo WITH path fixing' '\n+\tclient_view \"//depot/One/... //client/...\" &&\n+\tgit p4 clone --ignore-path-case --use-client-spec --destination=\"$git\" //depot/one &&\n+\ttest_when_finished cleanup_git &&\n+\t(\n+\t\tcd \"$git\" &&\n+\t\tcat >expect <<-\\EOF &&\n+\t\t\tTWO/File2.txt\n+\t\t\tTWO/file1.txt\n+\t\t\tTWO/file3.txt\n+\t\tEOF\n+\t\tgit ls-files >actual &&\n+\t\ttest_cmp expect actual\n+\t)\n+'\n+\n+# It looks like P4 determines the path case based on the first file in\n+# lexicographical order. Please note the lower case \"two\" directory for all\n+# files triggered through the addition of \"File0.txt\".\n+test_expect_success 'Add a new file and clone the repo WITH path fixing' '\n+\tclient_view \"//depot/... //client/...\" &&\n+\t(\n+\t\tcd \"$cli\" &&\n+\n+\t\tmkdir -p One/two &&\n+\t\t>One/two/File0.txt &&\n+\t\tp4 add One/two/File0.txt &&\n+\t\tp4 submit -d \"Add file\" &&\n+\t\trm -rf One\n+\t) &&\n+\n+\tclient_view \"//depot/One/... //client/...\" &&\n+\tgit p4 clone --ignore-path-case --use-client-spec --destination=\"$git\" //depot/one &&\n+\ttest_when_finished cleanup_git &&\n+\t(\n+\t\tcd \"$git\" &&\n+\t\tcat >expect <<-\\EOF &&\n+\t\t\ttwo/File0.txt\n+\t\t\ttwo/File2.txt\n+\t\t\ttwo/file1.txt\n+\t\t\ttwo/file3.txt\n+\t\tEOF\n+\t\tgit ls-files >actual &&\n+\t\ttest_cmp expect actual\n+\t)\n+'\n+\n+test_expect_success 'kill p4d' '\n+\tkill_p4d\n+'\n+\n+test_done\n--\n1.9.5 (Apple Git-50.3)\n"}]}