{"thread":{"id":"48558","subject":"[PATCHv6 0/1] git-p4: unshelve","startedAt":"2018-05-23T22:20:39Z","lastAt":"2018-05-23T22:20:44Z","messageCount":2,"participants":["Luke Diamand"],"isPatch":false,"patchVersion":null,"patchTotal":null},"messages":[{"id":"348393","messageId":"20180523222026.6609-1-luke@diamand.org","threadId":"48558","inReplyTo":null,"subject":"[PATCHv6 0/1] git-p4: unshelve","fromName":"Luke Diamand","fromEmail":"luke@diamand.org","sentAt":"2018-05-23T22:20:25Z","receivedAt":"2018-05-23T22:20:39Z","isPatch":false,"sender":{"key":"luke@diamand.org","avatar":"https://avatars.githubusercontent.com/u/5330967?v=4"},"body":"This just removes the verbose print change, which will end up causing\nconflicts later since it's also being fixed in another commit.\n\nDiscussed here:\n\nhttps://public-inbox.org/git/BYAPR08MB38455AFE85AE6B04EB31EF92DA6B0@BYAPR08MB3845.namprd08.prod.outlook.com/\n\nLuke Diamand (1):\n  git-p4: add unshelve command\n\n Documentation/git-p4.txt |  32 ++++++\n git-p4.py                | 213 ++++++++++++++++++++++++++++++++-------\n t/t9832-unshelve.sh      | 138 +++++++++++++++++++++++++\n 3 files changed, 347 insertions(+), 36 deletions(-)\n create mode 100755 t/t9832-unshelve.sh\n\n-- \n2.17.0.392.gdeb1a6e9b7\n\n"},{"id":"348394","messageId":"20180523222026.6609-2-luke@diamand.org","threadId":"48558","inReplyTo":"20180523222026.6609-1-luke@diamand.org","subject":"[PATCHv6 1/1] git-p4: add unshelve command","fromName":"Luke Diamand","fromEmail":"luke@diamand.org","sentAt":"2018-05-23T22:20:26Z","receivedAt":"2018-05-23T22:20:44Z","isPatch":false,"sender":{"key":"luke@diamand.org","avatar":"https://avatars.githubusercontent.com/u/5330967?v=4"},"body":"This can be used to \"unshelve\" a shelved P4 commit into\na git commit.\n\nFor example:\n\n  $ git p4 unshelve 12345\n\nThe resulting commit ends up in the branch:\n   refs/remotes/p4/unshelved/12345\n\nIf that branch already exists, it is renamed - for example\nthe above branch would be saved as p4/unshelved/12345.1.\n\ngit-p4 checks that the shelved changelist is based on files\nwhich are at the same Perforce revision as the origin branch\nbeing used for the unshelve (HEAD by default). If they are not,\nit will refuse to unshelve. This is to ensure that the unshelved\nchange does not contain other changes mixed-in.\n\nThe reference branch can be changed manually with the \"--origin\"\noption.\n\nThe change adds a new Unshelve command class. This just runs the\nexisting P4Sync code tweaked to handle a shelved changelist.\n\nSigned-off-by: Luke Diamand <luke@diamand.org>\n---\n Documentation/git-p4.txt |  32 ++++++\n git-p4.py                | 213 ++++++++++++++++++++++++++++++++-------\n t/t9832-unshelve.sh      | 138 +++++++++++++++++++++++++\n 3 files changed, 347 insertions(+), 36 deletions(-)\n create mode 100755 t/t9832-unshelve.sh\n\ndiff --git a/Documentation/git-p4.txt b/Documentation/git-p4.txt\nindex d8c8f11c9f..d3cb249fc2 100644\n--- a/Documentation/git-p4.txt\n+++ b/Documentation/git-p4.txt\n@@ -164,6 +164,31 @@ $ git p4 submit --shelve\n $ git p4 submit --update-shelve 1234 --update-shelve 2345\n ----\n \n+\n+Unshelve\n+~~~~~~~~\n+Unshelving will take a shelved P4 changelist, and produce the equivalent git commit\n+in the branch refs/remotes/p4/unshelved/<changelist>.\n+\n+The git commit is created relative to the current origin revision (HEAD by default).\n+If the shelved changelist's parent revisions differ, git-p4 will refuse to unshelve;\n+you need to be unshelving onto an equivalent tree.\n+\n+The origin revision can be changed with the \"--origin\" option.\n+\n+If the target branch in refs/remotes/p4/unshelved already exists, the old one will\n+be renamed.\n+\n+----\n+$ git p4 sync\n+$ git p4 unshelve 12345\n+$ git show refs/remotes/p4/unshelved/12345\n+<submit more changes via p4 to the same files>\n+$ git p4 unshelve 12345\n+<refuses to unshelve until git is in sync with p4 again>\n+\n+----\n+\n OPTIONS\n -------\n \n@@ -337,6 +362,13 @@ These options can be used to modify 'git p4 rebase' behavior.\n --import-labels::\n \tImport p4 labels.\n \n+Unshelve options\n+~~~~~~~~~~~~~~~~\n+\n+--origin::\n+    Sets the git refspec against which the shelved P4 changelist is compared.\n+    Defaults to p4/master.\n+\n DEPOT PATH SYNTAX\n -----------------\n The p4 depot path argument to 'git p4 sync' and 'git p4 clone' can\ndiff --git a/git-p4.py b/git-p4.py\nindex 7bb9cadc69..74d58afad3 100755\n--- a/git-p4.py\n+++ b/git-p4.py\n@@ -316,12 +316,17 @@ def p4_last_change():\n     results = p4CmdList([\"changes\", \"-m\", \"1\"], skip_info=True)\n     return int(results[0]['change'])\n \n-def p4_describe(change):\n+def p4_describe(change, shelved=False):\n     \"\"\"Make sure it returns a valid result by checking for\n        the presence of field \"time\".  Return a dict of the\n        results.\"\"\"\n \n-    ds = p4CmdList([\"describe\", \"-s\", str(change)], skip_info=True)\n+    cmd = [\"describe\", \"-s\"]\n+    if shelved:\n+        cmd += [\"-S\"]\n+    cmd += [str(change)]\n+\n+    ds = p4CmdList(cmd, skip_info=True)\n     if len(ds) != 1:\n         die(\"p4 describe -s %d did not return 1 result: %s\" % (change, str(ds)))\n \n@@ -662,6 +667,12 @@ def gitBranchExists(branch):\n                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);\n     return proc.wait() == 0;\n \n+def gitUpdateRef(ref, newvalue):\n+    subprocess.check_call([\"git\", \"update-ref\", ref, newvalue])\n+\n+def gitDeleteRef(ref):\n+    subprocess.check_call([\"git\", \"update-ref\", \"-d\", ref])\n+\n _gitConfig = {}\n \n def gitConfig(key, typeSpecifier=None):\n@@ -2411,6 +2422,7 @@ class P4Sync(Command, P4UserMap):\n         self.tempBranches = []\n         self.tempBranchLocation = \"refs/git-p4-tmp\"\n         self.largeFileSystem = None\n+        self.suppress_meta_comment = False\n \n         if gitConfig('git-p4.largeFileSystem'):\n             largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]\n@@ -2421,6 +2433,18 @@ class P4Sync(Command, P4UserMap):\n         if gitConfig(\"git-p4.syncFromOrigin\") == \"false\":\n             self.syncWithOrigin = False\n \n+        self.depotPaths = []\n+        self.changeRange = \"\"\n+        self.previousDepotPaths = []\n+        self.hasOrigin = False\n+\n+        # map from branch depot path to parent branch\n+        self.knownBranches = {}\n+        self.initialParents = {}\n+\n+        self.tz = \"%+03d%02d\" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))\n+        self.labels = {}\n+\n     # Force a checkpoint in fast-import and wait for it to finish\n     def checkpoint(self):\n         self.gitStream.write(\"checkpoint\\n\\n\")\n@@ -2429,7 +2453,20 @@ class P4Sync(Command, P4UserMap):\n         if self.verbose:\n             print \"checkpoint finished: \" + out\n \n-    def extractFilesFromCommit(self, commit):\n+    def cmp_shelved(self, path, filerev, revision):\n+        \"\"\" Determine if a path at revision #filerev is the same as the file\n+            at revision @revision for a shelved changelist. If they don't match,\n+            unshelving won't be safe (we will get other changes mixed in).\n+\n+            This is comparing the revision that the shelved changelist is *based* on, not\n+            the shelved changelist itself.\n+        \"\"\"\n+        ret = p4Cmd([\"diff2\", \"{0}#{1}\".format(path, filerev), \"{0}@{1}\".format(path, revision)])\n+        if verbose:\n+            print(\"p4 diff2 path %s filerev %s revision %s => %s\" % (path, filerev, revision, ret))\n+        return ret[\"status\"] == \"identical\"\n+\n+    def extractFilesFromCommit(self, commit, shelved=False, shelved_cl = 0, origin_revision = 0):\n         self.cloneExclude = [re.sub(r\"\\.\\.\\.$\", \"\", path)\n                              for path in self.cloneExclude]\n         files = []\n@@ -2452,6 +2489,19 @@ class P4Sync(Command, P4UserMap):\n             file[\"rev\"] = commit[\"rev%s\" % fnum]\n             file[\"action\"] = commit[\"action%s\" % fnum]\n             file[\"type\"] = commit[\"type%s\" % fnum]\n+            if shelved:\n+                file[\"shelved_cl\"] = int(shelved_cl)\n+\n+                # For shelved changelists, check that the revision of each file that the\n+                # shelve was based on matches the revision that we are using for the\n+                # starting point for git-fast-import (self.initialParent). Otherwise\n+                # the resulting diff will contain deltas from multiple commits.\n+\n+                if file[\"action\"] != \"add\" and \\\n+                    not self.cmp_shelved(path, file[\"rev\"], origin_revision):\n+                    sys.exit(\"change {0} not based on {1} for {2}, cannot unshelve\".format(\n+                        commit[\"change\"], self.initialParent, path))\n+\n             files.append(file)\n             fnum = fnum + 1\n         return files\n@@ -2743,7 +2793,16 @@ class P4Sync(Command, P4UserMap):\n             def streamP4FilesCbSelf(entry):\n                 self.streamP4FilesCb(entry)\n \n-            fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]\n+            fileArgs = []\n+            for f in filesToRead:\n+                if 'shelved_cl' in f:\n+                    # Handle shelved CLs using the \"p4 print file@=N\" syntax to print\n+                    # the contents\n+                    fileArg = '%s@=%d' % (f['path'], f['shelved_cl'])\n+                else:\n+                    fileArg = '%s#%s' % (f['path'], f['rev'])\n+\n+                fileArgs.append(fileArg)\n \n             p4CmdList([\"-x\", \"-\", \"print\"],\n                       stdin=fileArgs,\n@@ -2844,11 +2903,15 @@ class P4Sync(Command, P4UserMap):\n         self.gitStream.write(details[\"desc\"])\n         if len(jobs) > 0:\n             self.gitStream.write(\"\\nJobs: %s\" % (' '.join(jobs)))\n-        self.gitStream.write(\"\\n[git-p4: depot-paths = \\\"%s\\\": change = %s\" %\n-                             (','.join(self.branchPrefixes), details[\"change\"]))\n-        if len(details['options']) > 0:\n-            self.gitStream.write(\": options = %s\" % details['options'])\n-        self.gitStream.write(\"]\\nEOT\\n\\n\")\n+\n+        if not self.suppress_meta_comment:\n+            self.gitStream.write(\"\\n[git-p4: depot-paths = \\\"%s\\\": change = %s\" %\n+                                (','.join(self.branchPrefixes), details[\"change\"]))\n+            if len(details['options']) > 0:\n+                self.gitStream.write(\": options = %s\" % details['options'])\n+            self.gitStream.write(\"]\\n\")\n+\n+        self.gitStream.write(\"EOT\\n\\n\")\n \n         if len(parent) > 0:\n             if self.verbose:\n@@ -3162,10 +3225,10 @@ class P4Sync(Command, P4UserMap):\n         else:\n             return None\n \n-    def importChanges(self, changes):\n+    def importChanges(self, changes, shelved=False, origin_revision=0):\n         cnt = 1\n         for change in changes:\n-            description = p4_describe(change)\n+            description = p4_describe(change, shelved)\n             self.updateOptionDict(description)\n \n             if not self.silent:\n@@ -3235,7 +3298,7 @@ class P4Sync(Command, P4UserMap):\n                                 print \"Parent of %s not found. Committing into head of %s\" % (branch, parent)\n                             self.commit(description, filesForCommit, branch, parent)\n                 else:\n-                    files = self.extractFilesFromCommit(description)\n+                    files = self.extractFilesFromCommit(description, shelved, change, origin_revision)\n                     self.commit(description, files, self.branch,\n                                 self.initialParent)\n                     # only needed once, to connect to the previous commit\n@@ -3300,17 +3363,23 @@ class P4Sync(Command, P4UserMap):\n             print \"IO error with git fast-import. Is your git version recent enough?\"\n             print self.gitError.read()\n \n+    def openStreams(self):\n+        self.importProcess = subprocess.Popen([\"git\", \"fast-import\"],\n+                                              stdin=subprocess.PIPE,\n+                                              stdout=subprocess.PIPE,\n+                                              stderr=subprocess.PIPE);\n+        self.gitOutput = self.importProcess.stdout\n+        self.gitStream = self.importProcess.stdin\n+        self.gitError = self.importProcess.stderr\n \n-    def run(self, args):\n-        self.depotPaths = []\n-        self.changeRange = \"\"\n-        self.previousDepotPaths = []\n-        self.hasOrigin = False\n-\n-        # map from branch depot path to parent branch\n-        self.knownBranches = {}\n-        self.initialParents = {}\n+    def closeStreams(self):\n+        self.gitStream.close()\n+        if self.importProcess.wait() != 0:\n+            die(\"fast-import failed: %s\" % self.gitError.read())\n+        self.gitOutput.close()\n+        self.gitError.close()\n \n+    def run(self, args):\n         if self.importIntoRemotes:\n             self.refPrefix = \"refs/remotes/p4/\"\n         else:\n@@ -3497,15 +3566,7 @@ class P4Sync(Command, P4UserMap):\n                     b = b[len(self.projectName):]\n                 self.createdBranches.add(b)\n \n-        self.tz = \"%+03d%02d\" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))\n-\n-        self.importProcess = subprocess.Popen([\"git\", \"fast-import\"],\n-                                              stdin=subprocess.PIPE,\n-                                              stdout=subprocess.PIPE,\n-                                              stderr=subprocess.PIPE);\n-        self.gitOutput = self.importProcess.stdout\n-        self.gitStream = self.importProcess.stdin\n-        self.gitError = self.importProcess.stderr\n+        self.openStreams()\n \n         if revision:\n             self.importHeadRevision(revision)\n@@ -3585,11 +3646,7 @@ class P4Sync(Command, P4UserMap):\n             missingP4Labels = p4Labels - gitTags\n             self.importP4Labels(self.gitStream, missingP4Labels)\n \n-        self.gitStream.close()\n-        if self.importProcess.wait() != 0:\n-            die(\"fast-import failed: %s\" % self.gitError.read())\n-        self.gitOutput.close()\n-        self.gitError.close()\n+        self.closeStreams()\n \n         # Cleanup temporary branches created during import\n         if self.tempBranches != []:\n@@ -3721,6 +3778,89 @@ class P4Clone(P4Sync):\n \n         return True\n \n+class P4Unshelve(Command):\n+    def __init__(self):\n+        Command.__init__(self)\n+        self.options = []\n+        self.origin = \"HEAD\"\n+        self.description = \"Unshelve a P4 changelist into a git commit\"\n+        self.usage = \"usage: %prog [options] changelist\"\n+        self.options += [\n+                optparse.make_option(\"--origin\", dest=\"origin\",\n+                    help=\"Use this base revision instead of the default (%s)\" % self.origin),\n+        ]\n+        self.verbose = False\n+        self.noCommit = False\n+        self.destbranch = \"refs/remotes/p4/unshelved\"\n+\n+    def renameBranch(self, branch_name):\n+        \"\"\" Rename the existing branch to branch_name.N\n+        \"\"\"\n+\n+        found = True\n+        for i in range(0,1000):\n+            backup_branch_name = \"{0}.{1}\".format(branch_name, i)\n+            if not gitBranchExists(backup_branch_name):\n+                gitUpdateRef(backup_branch_name, branch_name) # copy ref to backup\n+                gitDeleteRef(branch_name)\n+                found = True\n+                print(\"renamed old unshelve branch to {0}\".format(backup_branch_name))\n+                break\n+\n+        if not found:\n+            sys.exit(\"gave up trying to rename existing branch {0}\".format(sync.branch))\n+\n+    def findLastP4Revision(self, starting_point):\n+        \"\"\" Look back from starting_point for the first commit created by git-p4\n+            to find the P4 commit we are based on, and the depot-paths.\n+        \"\"\"\n+\n+        for parent in (range(65535)):\n+            log = extractLogMessageFromGitCommit(\"{0}^{1}\".format(starting_point, parent))\n+            settings = extractSettingsGitLog(log)\n+            if settings.has_key('change'):\n+                return settings\n+\n+        sys.exit(\"could not find git-p4 commits in {0}\".format(self.origin))\n+\n+    def run(self, args):\n+        if len(args) != 1:\n+            return False\n+\n+        if not gitBranchExists(self.origin):\n+            sys.exit(\"origin branch {0} does not exist\".format(self.origin))\n+\n+        sync = P4Sync()\n+        changes = args\n+        sync.initialParent = self.origin\n+\n+        # use the first change in the list to construct the branch to unshelve into\n+        change = changes[0]\n+\n+        # if the target branch already exists, rename it\n+        branch_name = \"{0}/{1}\".format(self.destbranch, change)\n+        if gitBranchExists(branch_name):\n+            self.renameBranch(branch_name)\n+        sync.branch = branch_name\n+\n+        sync.verbose = self.verbose\n+        sync.suppress_meta_comment = True\n+\n+        settings = self.findLastP4Revision(self.origin)\n+        origin_revision = settings['change']\n+        sync.depotPaths = settings['depot-paths']\n+        sync.branchPrefixes = sync.depotPaths\n+\n+        sync.openStreams()\n+        sync.loadUserMapFromCache()\n+        sync.silent = True\n+        sync.importChanges(changes, shelved=True, origin_revision=origin_revision)\n+        sync.closeStreams()\n+\n+        print(\"unshelved changelist {0} into {1}\".format(change, branch_name))\n+\n+        return True\n+\n class P4Branches(Command):\n     def __init__(self):\n         Command.__init__(self)\n@@ -3775,7 +3915,8 @@ commands = {\n     \"rebase\" : P4Rebase,\n     \"clone\" : P4Clone,\n     \"rollback\" : P4RollBack,\n-    \"branches\" : P4Branches\n+    \"branches\" : P4Branches,\n+    \"unshelve\" : P4Unshelve,\n }\n \n \ndiff --git a/t/t9832-unshelve.sh b/t/t9832-unshelve.sh\nnew file mode 100755\nindex 0000000000..48ec7679b8\n--- /dev/null\n+++ b/t/t9832-unshelve.sh\n@@ -0,0 +1,138 @@\n+#!/bin/sh\n+\n+last_shelved_change () {\n+\tp4 changes -s shelved -m1 | cut -d \" \" -f 2\n+}\n+\n+test_description='git p4 unshelve'\n+\n+. ./lib-git-p4.sh\n+\n+test_expect_success 'start p4d' '\n+\tstart_p4d\n+'\n+\n+test_expect_success 'init depot' '\n+\t(\n+\t\tcd \"$cli\" &&\n+\t\techo file1 >file1 &&\n+\t\tp4 add file1 &&\n+\t\tp4 submit -d \"change 1\" &&\n+\t\t: >file_to_delete &&\n+\t\tp4 add file_to_delete &&\n+\t\tp4 submit -d \"file to delete\"\n+\t)\n+'\n+\n+test_expect_success 'initial clone' '\n+\tgit p4 clone --dest=\"$git\" //depot/@all\n+'\n+\n+test_expect_success 'create shelved changelist' '\n+\t(\n+\t\tcd \"$cli\" &&\n+\t\tp4 edit file1 &&\n+\t\techo \"a change\" >>file1 &&\n+\t\techo \"new file\" >file2 &&\n+\t\tp4 add file2 &&\n+\t\tp4 delete file_to_delete &&\n+\t\tp4 opened &&\n+\t\tp4 shelve -i <<EOF\n+Change: new\n+Description:\n+\tTest commit\n+\n+\tFurther description\n+Files:\n+\t//depot/file1\n+\t//depot/file2\n+\t//depot/file_to_delete\n+EOF\n+\n+\t) &&\n+\t(\n+\t\tcd \"$git\" &&\n+\t\tchange=$(last_shelved_change) &&\n+\t\tgit p4 unshelve $change &&\n+\t\tgit show refs/remotes/p4/unshelved/$change | grep -q \"Further description\" &&\n+\t\tgit cherry-pick refs/remotes/p4/unshelved/$change &&\n+\t\ttest_path_is_file file2 &&\n+\t\ttest_cmp file1 \"$cli\"/file1 &&\n+\t\ttest_cmp file2 \"$cli\"/file2 &&\n+\t\ttest_path_is_missing file_to_delete\n+\t)\n+'\n+\n+test_expect_success 'update shelved changelist and re-unshelve' '\n+\ttest_when_finished cleanup_git &&\n+\t(\n+\t\tcd \"$cli\" &&\n+\t\tchange=$(last_shelved_change) &&\n+\t\techo \"file3\" >file3 &&\n+\t\tp4 add -c $change file3 &&\n+\t\tp4 shelve -i -r <<EOF &&\n+Change: $change\n+Description:\n+\tTest commit\n+\n+\tFurther description\n+Files:\n+\t//depot/file1\n+\t//depot/file2\n+\t//depot/file3\n+\t//depot/file_to_delete\n+EOF\n+\t\tp4 describe $change\n+\t) &&\n+\t(\n+\t\tcd \"$git\" &&\n+\t\tchange=$(last_shelved_change) &&\n+\t\tgit p4 unshelve $change &&\n+\t\tgit diff refs/remotes/p4/unshelved/$change.0 refs/remotes/p4/unshelved/$change | grep -q file3\n+\t)\n+'\n+\n+# This is the tricky case where the shelved changelist base revision doesn't\n+# match git-p4's idea of the base revision\n+#\n+# We will attempt to unshelve a change that is based on a change one commit\n+# ahead of p4/master\n+\n+test_expect_success 'create shelved changelist based on p4 change ahead of p4/master' '\n+\tgit p4 clone --dest=\"$git\" //depot/@all &&\n+\t(\n+\t\tcd \"$cli\" &&\n+\t\tp4 revert ... &&\n+\t\tp4 edit file1 &&\n+\t\techo \"foo\" >>file1 &&\n+\t\tp4 submit -d \"change:foo\" &&\n+\t\tp4 edit file1 &&\n+\t\techo \"bar\" >>file1 &&\n+\t\tp4 shelve -i <<EOF &&\n+Change: new\n+Description:\n+\tChange to be unshelved\n+Files:\n+\t//depot/file1\n+EOF\n+\t\tchange=$(last_shelved_change) &&\n+\t\tp4 describe -S $change | grep -q \"Change to be unshelved\"\n+\t)\n+'\n+\n+# Now try to unshelve it. git-p4 should refuse to do so.\n+test_expect_success 'try to unshelve the change' '\n+\ttest_when_finished cleanup_git &&\n+\t(\n+\t\tchange=$(last_shelved_change) &&\n+\t\tcd \"$git\" &&\n+\t\ttest_must_fail git p4 unshelve $change 2>out.txt &&\n+\t\tgrep -q \"cannot unshelve\" out.txt\n+\t)\n+'\n+\n+test_expect_success 'kill p4d' '\n+\tkill_p4d\n+'\n+\n+test_done\n-- \n2.17.0.392.gdeb1a6e9b7\n\n"}]}