{"thread":{"id":"5996","subject":"[PATCH] RFC2047-encode email headers","startedAt":"2006-10-22T12:02:17Z","lastAt":"2006-10-22T12:49:37Z","messageCount":5,"participants":["Karl Hasselström"],"isPatch":true,"patchVersion":1,"patchTotal":null},"messages":[{"id":"29606","messageId":"20061022120217.7650.23715.stgit@localhost","threadId":"5996","inReplyTo":null,"subject":"[PATCH] RFC2047-encode email headers","fromName":"Karl Hasselström","fromEmail":"kha@treskal.com","sentAt":"2006-10-22T12:02:17Z","receivedAt":"2006-10-22T12:02:17Z","isPatch":true,"sender":{"key":"kha@treskal.com","avatar":"https://gravatar.com/avatar/f0120c734b5279b345075a28521e1ac66acb20c9913ffe9bf6ae97e53f7f3f13?d=mp&s=160"},"body":"From: Karl Hasselström <kha@treskal.com>\n\nHaving non-ascii characters in email headers is illegal, but StGIT\ncurrently does not care. I'm often bitten by this, since my name\ndoesn't fit in ascii.\n\nThis patch implements an encoding pass just before the email is sent\nover the wire -- in particular, it comes after any interactive editing\nand templates and such, so the user should never have to see the\nrfc2047 encoding.\n\nNOTE: The rfc2047 encoder needs to know the encoding of the input\nstring. This patch hard-codes this to utf8, since that should be by\nfar the most common non-ascii encoding, and since utf8 is already the\nhardcoded character set for the email body. In the long run, we\nprobably want to get this from the locale, or from a command line\nswitch, or both.\n\nSigned-off-by: Karl Hasselström <kha@treskal.com>\n---\n\n stgit/commands/mail.py |   45 +++++++++++++++++++++++++++++++++++++++++----\n 1 files changed, 41 insertions(+), 4 deletions(-)\n\ndiff --git a/stgit/commands/mail.py b/stgit/commands/mail.py\nindex 34504e6..b661308 100644\n--- a/stgit/commands/mail.py\n+++ b/stgit/commands/mail.py\n@@ -15,7 +15,7 @@ along with this program; if not, write t\n Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA\n \"\"\"\n \n-import sys, os, re, time, datetime, smtplib, email.Utils\n+import sys, os, re, time, datetime, smtplib, email.Header, email.Utils\n from optparse import OptionParser, make_option\n \n from stgit.commands.common import *\n@@ -403,6 +403,42 @@ def __build_message(tmpl, patch, patch_n\n \n     return msg.strip('\\n')\n \n+def encode_header(s, enc):\n+    \"\"\"Take an entire e-mail header line, encoded in enc, and\n+    rfc2047-encode it.\"\"\"\n+    def trans(s):\n+        return str(email.Header.Header(unicode(s, enc)))\n+    words = s.split(' ')\n+    first_encode = len(words)\n+    last_encode = -1\n+    for i in xrange(len(words)):\n+        ew = trans(words[i])\n+        if ew != words[i]:\n+            first_encode = min(first_encode, i)\n+            last_encode = max(last_encode, i)\n+    if first_encode <= last_encode:\n+        return ' '.join(filter(\n+            None,\n+            [' '.join(words[:first_encode]),\n+             trans(' '.join(words[first_encode:last_encode+1])),\n+             ' '.join(words[last_encode+1:])]))\n+    else:\n+        return s\n+\n+def encode_headers(msg, enc):\n+    \"\"\"rfc2047-encode the headers of msg, assuming it is encoded in\n+    enc.\"\"\"\n+    in_header = True\n+    lines = []\n+    for line in msg.splitlines(True):\n+        if in_header:\n+            if line.strip():\n+                line = encode_header(line, enc)\n+            else:\n+                in_header = False\n+        lines.append(line)\n+    return ''.join(lines)\n+\n def func(parser, options, args):\n     \"\"\"Send the patches by e-mail using the patchmail.tmpl file as\n     a template\n@@ -461,7 +497,8 @@ def func(parser, options, args):\n                 raise CmdException, 'No cover message template file found'\n \n         msg_id = email.Utils.make_msgid('stgit')\n-        msg = __build_cover(tmpl, total_nr, msg_id, options)\n+        msg = encode_headers(__build_cover(tmpl, total_nr, msg_id, options),\n+                             'UTF-8')\n         from_addr, to_addr_list = __parse_addresses(msg)\n \n         # subsequent e-mails are seen as replies to the first one\n@@ -487,8 +524,8 @@ def func(parser, options, args):\n \n     for (p, patch_nr) in zip(patches, range(1, len(patches) + 1)):\n         msg_id = email.Utils.make_msgid('stgit')\n-        msg = __build_message(tmpl, p, patch_nr, total_nr, msg_id, ref_id,\n-                              options)\n+        msg = encode_headers(__build_message(tmpl, p, patch_nr, total_nr,\n+                                             msg_id, ref_id, options), 'UTF-8')\n         from_addr, to_addr_list = __parse_addresses(msg)\n \n         # subsequent e-mails are seen as replies to the first one\n"},{"id":"29607","messageId":"20061022121240.GA21084@diana.vm.bytemark.co.uk","threadId":"5996","inReplyTo":"20061022120217.7650.23715.stgit@localhost","subject":"Re: [PATCH] RFC2047-encode email headers","fromName":"Karl Hasselström","fromEmail":"kha@treskal.com","sentAt":"2006-10-22T12:12:40Z","receivedAt":"2006-10-22T12:12:40Z","isPatch":true,"sender":{"key":"kha@treskal.com","avatar":"https://gravatar.com/avatar/f0120c734b5279b345075a28521e1ac66acb20c9913ffe9bf6ae97e53f7f3f13?d=mp&s=160"},"body":"On 2006-10-22 14:02:17 +0200, Karl Hasselström wrote:\n\n> From: Karl Hasselström <kha@treskal.com>\n\nOK, this patch did what it was supposed to do -- which was to encode\nthe mail headers properly -- but StGIT still generates an 8-bit\nencoded body, and vger doesn't seem to like that (see the X-Warning:\nheaders it added to the patch mail). That's another fix for another\nday.\n\nCatalin, if you take this patch, I'd appreciate it if you made double\nsure that my name doesn't have garbage in it. (It may very well be\nthat the copy of the patch sent to you personally is unharmed; it all\nworks fine when I send patches to myself. vger is the only mail server\nI have seen that has this problem.)\n\n-- \nKarl Hasselström, kha@treskal.com\n      www.treskal.com/kalle\n"},{"id":"29609","messageId":"20061022124551.14051.25145.stgit@localhost","threadId":"5996","inReplyTo":"20061022121240.GA21084@diana.vm.bytemark.co.uk","subject":"[PATCH 0/2] Make \"stg mail\" behave better with non-ascii characters","fromName":"Karl Hasselström","fromEmail":"kha@treskal.com","sentAt":"2006-10-22T12:45:51Z","receivedAt":"2006-10-22T12:45:51Z","isPatch":true,"sender":{"key":"kha@treskal.com","avatar":"https://gravatar.com/avatar/f0120c734b5279b345075a28521e1ac66acb20c9913ffe9bf6ae97e53f7f3f13?d=mp&s=160"},"body":"These two patches teach \"stg mail\" to ref2047-escape non-ascii\ncharacters in the mail headers (not doing so is illegal), and\nQP-encodes the body (leaving it as 8bit is not well received by some\nmail severs, notably vger).\n\nThe first patch is a resend, this time hopefully with my name intact.\n\n-- \nKarl Hasselström, kha@treskal.com\n      www.treskal.com/kalle\n"},{"id":"29610","messageId":"20061022124931.14051.39810.stgit@localhost","threadId":"5996","inReplyTo":"20061022124551.14051.25145.stgit@localhost","subject":"[PATCH 1/2] RFC2047-encode email headers","fromName":"Karl Hasselström","fromEmail":"kha@treskal.com","sentAt":"2006-10-22T12:49:32Z","receivedAt":"2006-10-22T12:49:32Z","isPatch":true,"sender":{"key":"kha@treskal.com","avatar":"https://gravatar.com/avatar/f0120c734b5279b345075a28521e1ac66acb20c9913ffe9bf6ae97e53f7f3f13?d=mp&s=160"},"body":"From: Karl Hasselström <kha@treskal.com>\n\nHaving non-ascii characters in email headers is illegal, but StGIT\ncurrently does not care. I'm often bitten by this, since my name\ndoesn't fit in ascii.\n\nThis patch implements an encoding pass just before the email is sent\nover the wire -- in particular, it comes after any interactive editing\nand templates and such, so the user should never have to see the\nrfc2047 encoding.\n\nNOTE: The rfc2047 encoder needs to know the encoding of the input\nstring. This patch hard-codes this to utf8, since that should be by\nfar the most common non-ascii encoding, and since utf8 is already the\nhardcoded character set for the email body. In the long run, we\nprobably want to get this from the locale, or from a command line\nswitch, or both.\n\nSigned-off-by: Karl Hasselström <kha@treskal.com>\n---\n\n stgit/commands/mail.py |   45 +++++++++++++++++++++++++++++++++++++++++----\n 1 files changed, 41 insertions(+), 4 deletions(-)\n\ndiff --git a/stgit/commands/mail.py b/stgit/commands/mail.py\nindex 34504e6..b661308 100644\n--- a/stgit/commands/mail.py\n+++ b/stgit/commands/mail.py\n@@ -15,7 +15,7 @@ along with this program; if not, write t\n Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA\n \"\"\"\n \n-import sys, os, re, time, datetime, smtplib, email.Utils\n+import sys, os, re, time, datetime, smtplib, email.Header, email.Utils\n from optparse import OptionParser, make_option\n \n from stgit.commands.common import *\n@@ -403,6 +403,42 @@ def __build_message(tmpl, patch, patch_n\n \n     return msg.strip('\\n')\n \n+def encode_header(s, enc):\n+    \"\"\"Take an entire e-mail header line, encoded in enc, and\n+    rfc2047-encode it.\"\"\"\n+    def trans(s):\n+        return str(email.Header.Header(unicode(s, enc)))\n+    words = s.split(' ')\n+    first_encode = len(words)\n+    last_encode = -1\n+    for i in xrange(len(words)):\n+        ew = trans(words[i])\n+        if ew != words[i]:\n+            first_encode = min(first_encode, i)\n+            last_encode = max(last_encode, i)\n+    if first_encode <= last_encode:\n+        return ' '.join(filter(\n+            None,\n+            [' '.join(words[:first_encode]),\n+             trans(' '.join(words[first_encode:last_encode+1])),\n+             ' '.join(words[last_encode+1:])]))\n+    else:\n+        return s\n+\n+def encode_headers(msg, enc):\n+    \"\"\"rfc2047-encode the headers of msg, assuming it is encoded in\n+    enc.\"\"\"\n+    in_header = True\n+    lines = []\n+    for line in msg.splitlines(True):\n+        if in_header:\n+            if line.strip():\n+                line = encode_header(line, enc)\n+            else:\n+                in_header = False\n+        lines.append(line)\n+    return ''.join(lines)\n+\n def func(parser, options, args):\n     \"\"\"Send the patches by e-mail using the patchmail.tmpl file as\n     a template\n@@ -461,7 +497,8 @@ def func(parser, options, args):\n                 raise CmdException, 'No cover message template file found'\n \n         msg_id = email.Utils.make_msgid('stgit')\n-        msg = __build_cover(tmpl, total_nr, msg_id, options)\n+        msg = encode_headers(__build_cover(tmpl, total_nr, msg_id, options),\n+                             'UTF-8')\n         from_addr, to_addr_list = __parse_addresses(msg)\n \n         # subsequent e-mails are seen as replies to the first one\n@@ -487,8 +524,8 @@ def func(parser, options, args):\n \n     for (p, patch_nr) in zip(patches, range(1, len(patches) + 1)):\n         msg_id = email.Utils.make_msgid('stgit')\n-        msg = __build_message(tmpl, p, patch_nr, total_nr, msg_id, ref_id,\n-                              options)\n+        msg = encode_headers(__build_message(tmpl, p, patch_nr, total_nr,\n+                                             msg_id, ref_id, options), 'UTF-8')\n         from_addr, to_addr_list = __parse_addresses(msg)\n \n         # subsequent e-mails are seen as replies to the first one\n"},{"id":"29611","messageId":"20061022124937.14051.37985.stgit@localhost","threadId":"5996","inReplyTo":"20061022124551.14051.25145.stgit@localhost","subject":"[PATCH 2/2] QP-encode email body","fromName":"Karl Hasselström","fromEmail":"kha@treskal.com","sentAt":"2006-10-22T12:49:37Z","receivedAt":"2006-10-22T12:49:37Z","isPatch":true,"sender":{"key":"kha@treskal.com","avatar":"https://gravatar.com/avatar/f0120c734b5279b345075a28521e1ac66acb20c9913ffe9bf6ae97e53f7f3f13?d=mp&s=160"},"body":"From: Karl Hasselström <kha@treskal.com>\n\nSome mail servers dislike the 8bit transfer encoding, so use\nquoted-printable instead.\n\nSigned-off-by: Karl Hasselström <kha@treskal.com>\n---\n\n stgit/commands/mail.py |   16 +++++++++-------\n 1 files changed, 9 insertions(+), 7 deletions(-)\n\ndiff --git a/stgit/commands/mail.py b/stgit/commands/mail.py\nindex b661308..885d5e9 100644\n--- a/stgit/commands/mail.py\n+++ b/stgit/commands/mail.py\n@@ -15,7 +15,7 @@ along with this program; if not, write t\n Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA\n \"\"\"\n \n-import sys, os, re, time, datetime, smtplib, email.Header, email.Utils\n+import sys, os, re, time, datetime, quopri, smtplib, email.Header, email.Utils\n from optparse import OptionParser, make_option\n \n from stgit.commands.common import *\n@@ -253,7 +253,7 @@ def __build_extra_headers():\n     \"\"\"Build extra headers like content-type etc.\n     \"\"\"\n     headers  = 'Content-Type: text/plain; charset=utf-8; format=fixed\\n'\n-    headers += 'Content-Transfer-Encoding: 8bit\\n'\n+    headers += 'Content-Transfer-Encoding: quoted-printable\\n'\n     headers += 'User-Agent: StGIT/%s\\n' % version.version\n \n     return headers\n@@ -425,9 +425,9 @@ def encode_header(s, enc):\n     else:\n         return s\n \n-def encode_headers(msg, enc):\n-    \"\"\"rfc2047-encode the headers of msg, assuming it is encoded in\n-    enc.\"\"\"\n+def encode_message(msg, enc):\n+    \"\"\"rfc2047-encode the headers of msg, and quoted-printable-encode\n+    the body. msg is assumed to be encoded in enc.\"\"\"\n     in_header = True\n     lines = []\n     for line in msg.splitlines(True):\n@@ -436,6 +436,8 @@ def encode_headers(msg, enc):\n                 line = encode_header(line, enc)\n             else:\n                 in_header = False\n+        else:\n+            line = quopri.encodestring(line)\n         lines.append(line)\n     return ''.join(lines)\n \n@@ -497,7 +499,7 @@ def func(parser, options, args):\n                 raise CmdException, 'No cover message template file found'\n \n         msg_id = email.Utils.make_msgid('stgit')\n-        msg = encode_headers(__build_cover(tmpl, total_nr, msg_id, options),\n+        msg = encode_message(__build_cover(tmpl, total_nr, msg_id, options),\n                              'UTF-8')\n         from_addr, to_addr_list = __parse_addresses(msg)\n \n@@ -524,7 +526,7 @@ def func(parser, options, args):\n \n     for (p, patch_nr) in zip(patches, range(1, len(patches) + 1)):\n         msg_id = email.Utils.make_msgid('stgit')\n-        msg = encode_headers(__build_message(tmpl, p, patch_nr, total_nr,\n+        msg = encode_message(__build_message(tmpl, p, patch_nr, total_nr,\n                                              msg_id, ref_id, options), 'UTF-8')\n         from_addr, to_addr_list = __parse_addresses(msg)\n \n"}]}