{"thread":{"id":"40233","subject":"[PATCH v2] git-p4: add \"--path-encoding\" option","startedAt":"2015-08-31T20:43:52Z","lastAt":"2015-08-31T21:10:39Z","messageCount":4,"participants":["larsxschneider@gmail.com","Junio C Hamano"],"isPatch":true,"patchVersion":2,"patchTotal":null},"messages":[{"id":"269048","messageId":"1441053833-63790-1-git-send-email-larsxschneider@gmail.com","threadId":"40233","inReplyTo":null,"subject":"[PATCH v2] git-p4: add \"--path-encoding\" option","fromName":"","fromEmail":"larsxschneider@gmail.com","sentAt":"2015-08-31T20:43:52Z","receivedAt":"2015-08-31T20:43:52Z","isPatch":true,"sender":{"key":"larsxschneider@gmail.com","avatar":"https://avatars.githubusercontent.com/u/477434?v=4"},"body":"From: Lars Schneider <larsxschneider@gmail.com>\n\nDiff to v1:\n* switch example conversions from cp1252 to iso8859-1 (thanks Torsten!)\n* fix git-p4.txt line length and double dashes (thanks Junio!)\n* remove bare UTF-8 sequence (thanks Junio!)\n\nAs with v1, I ensured the unit test runs on OS X and Linux.\n\nI noticed one weird point, though. \"git ls-files\" outputs the UTF-8 characters escaped on Linux and on OS X. Is there a problem with my setup or this a Git bug?\n\nThanks,\nLars\n\nLars Schneider (1):\n  git-p4: add \"--path-encoding\" option\n\n Documentation/git-p4.txt        |  5 +++++\n git-p4.py                       |  6 ++++++\n t/t9821-git-p4-path-encoding.sh | 39 +++++++++++++++++++++++++++++++++++++++\n 3 files changed, 50 insertions(+)\n create mode 100755 t/t9821-git-p4-path-encoding.sh\n\n--\n2.5.1.1.g9071995.dirty\n"},{"id":"269049","messageId":"1441053833-63790-2-git-send-email-larsxschneider@gmail.com","threadId":"40233","inReplyTo":"1441053833-63790-1-git-send-email-larsxschneider@gmail.com","subject":"[PATCH v2] git-p4: add \"--path-encoding\" option","fromName":"","fromEmail":"larsxschneider@gmail.com","sentAt":"2015-08-31T20:43:53Z","receivedAt":"2015-08-31T20:43:53Z","isPatch":true,"sender":{"key":"larsxschneider@gmail.com","avatar":"https://avatars.githubusercontent.com/u/477434?v=4"},"body":"From: Lars Schneider <larsxschneider@gmail.com>\n\nSigned-off-by: Lars Schneider <larsxschneider@gmail.com>\n---\n Documentation/git-p4.txt        |  5 +++++\n git-p4.py                       |  6 ++++++\n t/t9821-git-p4-path-encoding.sh | 39 +++++++++++++++++++++++++++++++++++++++\n 3 files changed, 50 insertions(+)\n create mode 100755 t/t9821-git-p4-path-encoding.sh\n\ndiff --git a/Documentation/git-p4.txt b/Documentation/git-p4.txt\nindex 82aa5d6..14bb79c 100644\n--- a/Documentation/git-p4.txt\n+++ b/Documentation/git-p4.txt\n@@ -252,6 +252,11 @@ Git repository:\n \tUse a client spec to find the list of interesting files in p4.\n \tSee the \"CLIENT SPEC\" section below.\n \n+--path-encoding <encoding>::\n+\tThe encoding to use when reading p4 client paths. With this option\n+\tnon ASCII paths are properly stored in Git. For example, the encoding\n+\t'cp1252' is often used on Windows systems.\n+\n -/ <path>::\n \tExclude selected depot paths when cloning or syncing.\n \ndiff --git a/git-p4.py b/git-p4.py\nindex 073f87b..2b3bfc4 100755\n--- a/git-p4.py\n+++ b/git-p4.py\n@@ -1981,6 +1981,8 @@ class P4Sync(Command, P4UserMap):\n                 optparse.make_option(\"--silent\", dest=\"silent\", action=\"store_true\"),\n                 optparse.make_option(\"--detect-labels\", dest=\"detectLabels\", action=\"store_true\"),\n                 optparse.make_option(\"--import-labels\", dest=\"importLabels\", action=\"store_true\"),\n+                optparse.make_option(\"--path-encoding\", dest=\"pathEncoding\", type=\"string\",\n+                                     help=\"Encoding to use for paths\"),\n                 optparse.make_option(\"--import-local\", dest=\"importIntoRemotes\", action=\"store_false\",\n                                      help=\"Import into refs/heads/ , not refs/remotes\"),\n                 optparse.make_option(\"--max-changes\", dest=\"maxChanges\",\n@@ -2025,6 +2027,7 @@ class P4Sync(Command, P4UserMap):\n         self.clientSpecDirs = None\n         self.tempBranches = []\n         self.tempBranchLocation = \"git-p4-tmp\"\n+        self.pathEncoding = None\n \n         if gitConfig(\"git-p4.syncFromOrigin\") == \"false\":\n             self.syncWithOrigin = False\n@@ -2213,6 +2216,9 @@ class P4Sync(Command, P4UserMap):\n             text = regexp.sub(r'$\\1$', text)\n             contents = [ text ]\n \n+        if self.pathEncoding:\n+            relPath = relPath.decode(self.pathEncoding).encode('utf8', 'replace')\n+\n         self.gitStream.write(\"M %s inline %s\\n\" % (git_mode, relPath))\n \n         # total length...\ndiff --git a/t/t9821-git-p4-path-encoding.sh b/t/t9821-git-p4-path-encoding.sh\nnew file mode 100755\nindex 0000000..bb85074\n--- /dev/null\n+++ b/t/t9821-git-p4-path-encoding.sh\n@@ -0,0 +1,39 @@\n+#!/bin/sh\n+\n+test_description='Clone repositories with non ASCII paths'\n+\n+. ./lib-git-p4.sh\n+\n+UTF8_ESCAPED=\"a-\\303\\244_o-\\303\\266_u-\\303\\274.txt\"\n+\n+test_expect_success 'start p4d' '\n+\tstart_p4d\n+'\n+\n+test_expect_success 'Create a repo containing iso8859-1 encoded paths' '\n+\tcd \"$cli\" &&\n+\n+\tISO8859=\"$(printf \"$UTF8_ESCAPED\" | iconv -f utf-8 -t iso8859-1)\" &&\n+\t>\"$ISO8859\" &&\n+\tp4 add \"$ISO8859\" &&\n+\tp4 submit -d \"test commit\"\n+'\n+\n+test_expect_success 'Clone repo containing iso8859-1 encoded paths' '\n+\tgit p4 clone --destination=\"$git\" --path-encoding=iso8859-1 //depot &&\n+\ttest_when_finished cleanup_git &&\n+\t(\n+\t\tcd \"$git\" &&\n+\t\tprintf \"\\\"$UTF8_ESCAPED\\\"\" >expect &&\n+\t\t# I wonder why Git prints \"ls-files\" output UTF-8 escaped.\n+\t\t# This behavior is consistent on Linux and OS X.\n+\t\tprintf $(git ls-files) >actual &&\n+\t\ttest_cmp expect actual\n+\t)\n+'\n+\n+test_expect_success 'kill p4d' '\n+\tkill_p4d\n+'\n+\n+test_done\n-- \n2.5.1.1.g9071995.dirty\n"},{"id":"269054","messageId":"xmqqsi6ztbcj.fsf@gitster.mtv.corp.google.com","threadId":"40233","inReplyTo":"1441053833-63790-1-git-send-email-larsxschneider@gmail.com","subject":"Re: [PATCH v2] git-p4: add \"--path-encoding\" option","fromName":"Junio C Hamano","fromEmail":"gitster@pobox.com","sentAt":"2015-08-31T21:08:28Z","receivedAt":"2015-08-31T21:08:28Z","isPatch":true,"sender":{"key":"gitster@pobox.com","avatar":"https://avatars.githubusercontent.com/u/54884?v=4"},"body":"larsxschneider@gmail.com writes:\n\n> From: Lars Schneider <larsxschneider@gmail.com>\n>\n> Diff to v1:\n> * switch example conversions from cp1252 to iso8859-1 (thanks Torsten!)\n> * fix git-p4.txt line length and double dashes (thanks Junio!)\n> * remove bare UTF-8 sequence (thanks Junio!)\n>\n> As with v1, I ensured the unit test runs on OS X and Linux.\n>\n> I noticed one weird point, though. \"git ls-files\" outputs the UTF-8 characters escaped on Linux and on OS X. Is there a problem with my setup or this a Git bug?\n\nThere is no bug, there is no misconfiguration on your part.  It is\nvery much deliberate, I think.  core.quotepath defaults to false.\n\nAsking is very good, but please don't do so in in-code comment.\n"},{"id":"269055","messageId":"xmqqoahntb8w.fsf@gitster.mtv.corp.google.com","threadId":"40233","inReplyTo":"1441053833-63790-2-git-send-email-larsxschneider@gmail.com","subject":"Re: [PATCH v2] git-p4: add \"--path-encoding\" option","fromName":"Junio C Hamano","fromEmail":"gitster@pobox.com","sentAt":"2015-08-31T21:10:39Z","receivedAt":"2015-08-31T21:10:39Z","isPatch":true,"sender":{"key":"gitster@pobox.com","avatar":"https://avatars.githubusercontent.com/u/54884?v=4"},"body":"larsxschneider@gmail.com writes:\n\n> From: Lars Schneider <larsxschneider@gmail.com>\n>\n> Signed-off-by: Lars Schneider <larsxschneider@gmail.com>\n> ---\n>  Documentation/git-p4.txt        |  5 +++++\n>  git-p4.py                       |  6 ++++++\n>  t/t9821-git-p4-path-encoding.sh | 39 +++++++++++++++++++++++++++++++++++++++\n>  3 files changed, 50 insertions(+)\n>  create mode 100755 t/t9821-git-p4-path-encoding.sh\n>\n> diff --git a/Documentation/git-p4.txt b/Documentation/git-p4.txt\n> index 82aa5d6..14bb79c 100644\n> --- a/Documentation/git-p4.txt\n> +++ b/Documentation/git-p4.txt\n> @@ -252,6 +252,11 @@ Git repository:\n>  \tUse a client spec to find the list of interesting files in p4.\n>  \tSee the \"CLIENT SPEC\" section below.\n>  \n> +--path-encoding <encoding>::\n> +\tThe encoding to use when reading p4 client paths. With this option\n> +\tnon ASCII paths are properly stored in Git. For example, the encoding\n> +\t'cp1252' is often used on Windows systems.\n> +\n>  -/ <path>::\n>  \tExclude selected depot paths when cloning or syncing.\n>  \n> diff --git a/git-p4.py b/git-p4.py\n> index 073f87b..2b3bfc4 100755\n> --- a/git-p4.py\n> +++ b/git-p4.py\n> @@ -1981,6 +1981,8 @@ class P4Sync(Command, P4UserMap):\n>                  optparse.make_option(\"--silent\", dest=\"silent\", action=\"store_true\"),\n>                  optparse.make_option(\"--detect-labels\", dest=\"detectLabels\", action=\"store_true\"),\n>                  optparse.make_option(\"--import-labels\", dest=\"importLabels\", action=\"store_true\"),\n> +                optparse.make_option(\"--path-encoding\", dest=\"pathEncoding\", type=\"string\",\n> +                                     help=\"Encoding to use for paths\"),\n>                  optparse.make_option(\"--import-local\", dest=\"importIntoRemotes\", action=\"store_false\",\n>                                       help=\"Import into refs/heads/ , not refs/remotes\"),\n>                  optparse.make_option(\"--max-changes\", dest=\"maxChanges\",\n> @@ -2025,6 +2027,7 @@ class P4Sync(Command, P4UserMap):\n>          self.clientSpecDirs = None\n>          self.tempBranches = []\n>          self.tempBranchLocation = \"git-p4-tmp\"\n> +        self.pathEncoding = None\n>  \n>          if gitConfig(\"git-p4.syncFromOrigin\") == \"false\":\n>              self.syncWithOrigin = False\n> @@ -2213,6 +2216,9 @@ class P4Sync(Command, P4UserMap):\n>              text = regexp.sub(r'$\\1$', text)\n>              contents = [ text ]\n>  \n> +        if self.pathEncoding:\n> +            relPath = relPath.decode(self.pathEncoding).encode('utf8', 'replace')\n> +\n>          self.gitStream.write(\"M %s inline %s\\n\" % (git_mode, relPath))\n>  \n>          # total length...\n> diff --git a/t/t9821-git-p4-path-encoding.sh b/t/t9821-git-p4-path-encoding.sh\n> new file mode 100755\n> index 0000000..bb85074\n> --- /dev/null\n> +++ b/t/t9821-git-p4-path-encoding.sh\n> @@ -0,0 +1,39 @@\n> +#!/bin/sh\n> +\n> +test_description='Clone repositories with non ASCII paths'\n> +\n> +. ./lib-git-p4.sh\n> +\n> +UTF8_ESCAPED=\"a-\\303\\244_o-\\303\\266_u-\\303\\274.txt\"\n> +\n> +test_expect_success 'start p4d' '\n> +\tstart_p4d\n> +'\n> +\n> +test_expect_success 'Create a repo containing iso8859-1 encoded paths' '\n> +\tcd \"$cli\" &&\n> +\n> +\tISO8859=\"$(printf \"$UTF8_ESCAPED\" | iconv -f utf-8 -t iso8859-1)\" &&\n> +\t>\"$ISO8859\" &&\n> +\tp4 add \"$ISO8859\" &&\n> +\tp4 submit -d \"test commit\"\n> +'\n> +\n> +test_expect_success 'Clone repo containing iso8859-1 encoded paths' '\n> +\tgit p4 clone --destination=\"$git\" --path-encoding=iso8859-1 //depot &&\n> +\ttest_when_finished cleanup_git &&\n> +\t(\n> +\t\tcd \"$git\" &&\n> +\t\tprintf \"\\\"$UTF8_ESCAPED\\\"\" >expect &&\n\nDid this pass your test?  I suspect 'expect' lacks the final LF.\n\n> +\t\t# I wonder why Git prints \"ls-files\" output UTF-8 escaped.\n> +\t\t# This behavior is consistent on Linux and OS X.\n> +\t\tprintf $(git ls-files) >actual &&\n\nYuck.  Perhaps do it like so:\n\n\ttest_config core.quotepath false &&\n\tgit ls-files >actual &&\n        test_cmp expect actual\n\n> +\t\ttest_cmp expect actual\n> +\t)\n> +'\n> +\n> +test_expect_success 'kill p4d' '\n> +\tkill_p4d\n> +'\n> +\n> +test_done\n"}]}