diff mbox

[8/9] py3: Resolve unicode issues

Message ID 1448835050-1572-9-git-send-email-stephen.finucane@intel.com
State Accepted
Headers show

Commit Message

Stephen Finucane Nov. 29, 2015, 10:10 p.m. UTC
Python 3 is unicode only. While many of the issues with unicode, such
as the now invalid 'u' prefix, have already been resolved, there are a
few more issues.

Many of these issues are related to HTTPResponse.content, which returns
bytes and needs to be "decoded" in order to perform actions like
concatenation with str objects (unicode). Where possible, make use of
assertContains, per the Django documentation (http://bit.ly/1lRDYie),
else fall back to including a 'decode' statement.

Signed-off-by: Stephen Finucane <stephen.finucane@intel.com>
---
 patchwork/bin/parsemail.py            |  4 +++-
 patchwork/parser.py                   |  2 +-
 patchwork/tests/test_bundles.py       |  4 ++--
 patchwork/tests/test_filters.py       |  4 ++--
 patchwork/tests/test_list.py          |  2 +-
 patchwork/tests/test_mail_settings.py | 27 +++++++++++++--------------
 patchwork/tests/test_mboxviews.py     |  4 ++--
 patchwork/tests/test_person.py        |  6 +++---
 patchwork/views/patch.py              |  7 ++++++-
 9 files changed, 33 insertions(+), 27 deletions(-)

Comments

Stephen Finucane Dec. 10, 2015, 8:21 p.m. UTC | #1
On 29 Nov 22:10, Stephen Finucane wrote:
> Python 3 is unicode only. While many of the issues with unicode, such
> as the now invalid 'u' prefix, have already been resolved, there are a
> few more issues.
> 
> Many of these issues are related to HTTPResponse.content, which returns
> bytes and needs to be "decoded" in order to perform actions like
> concatenation with str objects (unicode). Where possible, make use of
> assertContains, per the Django documentation (http://bit.ly/1lRDYie),
> else fall back to including a 'decode' statement.
> 
> Signed-off-by: Stephen Finucane <stephen.finucane@intel.com>

I have a feeling these issues may pop up again, but current Python 2.7
installations shouldn't be affected. Merged.
diff mbox

Patch

diff --git a/patchwork/bin/parsemail.py b/patchwork/bin/parsemail.py
index 2bc080c..43d733f 100755
--- a/patchwork/bin/parsemail.py
+++ b/patchwork/bin/parsemail.py
@@ -68,7 +68,9 @@  def clean_header(header):
         (frag_str, frag_encoding) = fragment
         if frag_encoding:
             return frag_str.decode(frag_encoding)
-        return frag_str.decode()
+        elif isinstance(frag_str, six.binary_type):  # python 2
+            return frag_str.decode()
+        return frag_str
 
     fragments = list(map(decode, decode_header(header)))
 
diff --git a/patchwork/parser.py b/patchwork/parser.py
index 84f704a..b65e50c 100644
--- a/patchwork/parser.py
+++ b/patchwork/parser.py
@@ -230,7 +230,7 @@  def hash_patch(str):
             # other lines are ignored
             continue
 
-        hash.update(line.encode('utf-8') + '\n')
+        hash.update((line + '\n').encode('utf-8'))
 
     return hash
 
diff --git a/patchwork/tests/test_bundles.py b/patchwork/tests/test_bundles.py
index 4b982a5..d5d5d2f 100644
--- a/patchwork/tests/test_bundles.py
+++ b/patchwork/tests/test_bundles.py
@@ -109,7 +109,7 @@  class BundleViewTest(BundleTestBase):
 
         pos = 0
         for patch in self.patches:
-            next_pos = response.content.find(patch.name)
+            next_pos = response.content.decode().find(patch.name)
             # ensure that this patch is after the previous
             self.failUnless(next_pos > pos)
             pos = next_pos
@@ -126,7 +126,7 @@  class BundleViewTest(BundleTestBase):
         response = self.client.get(bundle_url(self.bundle))
         pos = len(response.content)
         for patch in self.patches:
-            next_pos = response.content.find(patch.name)
+            next_pos = response.content.decode().find(patch.name)
             # ensure that this patch is now *before* the previous
             self.failUnless(next_pos < pos)
             pos = next_pos
diff --git a/patchwork/tests/test_filters.py b/patchwork/tests/test_filters.py
index ae1ff7c..c92146b 100644
--- a/patchwork/tests/test_filters.py
+++ b/patchwork/tests/test_filters.py
@@ -35,8 +35,8 @@  class FilterQueryStringTest(TestCase):
         url = '/project/%s/list/?submitter=a%%26b=c' % project.linkname
         response = self.client.get(url)
         self.failUnlessEqual(response.status_code, 200)
-        self.failIf('submitter=a&amp;b=c' in response.content)
-        self.failIf('submitter=a&b=c' in response.content)
+        self.assertNotContains(response, 'submitter=a&amp;b=c')
+        self.assertNotContains(response, 'submitter=a&b=c')
 
     def testUTF8QSHandling(self):
         """test that non-ascii characters can be handled by the filter
diff --git a/patchwork/tests/test_list.py b/patchwork/tests/test_list.py
index 0f23d13..62ff121 100644
--- a/patchwork/tests/test_list.py
+++ b/patchwork/tests/test_list.py
@@ -77,7 +77,7 @@  class PatchOrderTest(TestCase):
 
     def _extract_patch_ids(self, response):
         id_re = re.compile('<tr id="patch_row:(\d+)"')
-        ids = [ int(m.group(1)) for m in id_re.finditer(response.content) ]
+        ids = [ int(m.group(1)) for m in id_re.finditer(response.content.decode()) ]
         return ids
 
     def _test_sequence(self, response, test_fn):
diff --git a/patchwork/tests/test_mail_settings.py b/patchwork/tests/test_mail_settings.py
index a3956a9..39133f9 100644
--- a/patchwork/tests/test_mail_settings.py
+++ b/patchwork/tests/test_mail_settings.py
@@ -63,9 +63,9 @@  class MailSettingsTest(TestCase):
         self.assertEquals(response.status_code, 200)
         self.assertTemplateUsed(response, 'patchwork/mail-settings.html')
         self.assertEquals(response.context['is_optout'], False)
-        self.assertTrue('<strong>may</strong>' in response.content)
+        self.assertContains(response, '<strong>may</strong>')
         optout_url = reverse('patchwork.views.mail.optout')
-        self.assertTrue(('action="%s"' % optout_url) in response.content)
+        self.assertContains(response, ('action="%s"' % optout_url))
 
     def testMailSettingsPOSTOptedOut(self):
         email = u'foo@example.com'
@@ -74,9 +74,9 @@  class MailSettingsTest(TestCase):
         self.assertEquals(response.status_code, 200)
         self.assertTemplateUsed(response, 'patchwork/mail-settings.html')
         self.assertEquals(response.context['is_optout'], True)
-        self.assertTrue('<strong>may not</strong>' in response.content)
+        self.assertContains(response, '<strong>may not</strong>')
         optin_url = reverse('patchwork.views.mail.optin')
-        self.assertTrue(('action="%s"' % optin_url) in response.content)
+        self.assertContains(response, ('action="%s"' % optin_url))
 
 class OptoutRequestTest(TestCase):
 
@@ -98,7 +98,7 @@  class OptoutRequestTest(TestCase):
         # check confirmation page
         self.assertEquals(response.status_code, 200)
         self.assertEquals(response.context['confirmation'], conf)
-        self.assertTrue(email in response.content)
+        self.assertContains(response, email)
 
         # check email
         url = reverse('patchwork.views.confirm', kwargs = {'key': conf.key})
@@ -140,7 +140,7 @@  class OptoutTest(TestCase):
 
         self.assertEquals(response.status_code, 200)
         self.assertTemplateUsed(response, 'patchwork/optout.html')
-        self.assertTrue(self.email in response.content)
+        self.assertContains(response, self.email)
 
         # check that we've got an optout in the list
         self.assertEquals(EmailOptout.objects.count(), 1)
@@ -178,7 +178,7 @@  class OptinRequestTest(TestCase):
         # check confirmation page
         self.assertEquals(response.status_code, 200)
         self.assertEquals(response.context['confirmation'], conf)
-        self.assertTrue(self.email in response.content)
+        self.assertContains(response, self.email)
 
         # check email
         url = reverse('patchwork.views.confirm', kwargs = {'key': conf.key})
@@ -221,7 +221,7 @@  class OptinTest(TestCase):
 
         self.assertEquals(response.status_code, 200)
         self.assertTemplateUsed(response, 'patchwork/optin.html')
-        self.assertTrue(self.email in response.content)
+        self.assertContains(response, self.email)
 
         # check that there's no optout remaining
         self.assertEquals(EmailOptout.objects.count(), 0)
@@ -243,7 +243,7 @@  class OptinWithoutOptoutTest(TestCase):
         # check for an error message
         self.assertEquals(response.status_code, 200)
         self.assertTrue(bool(response.context['error']))
-        self.assertTrue('not on the patchwork opt-out list' in response.content)
+        self.assertContains(response, 'not on the patchwork opt-out list')
 
 class UserProfileOptoutFormTest(TestCase):
     """Test that the correct optin/optout forms appear on the user profile
@@ -270,23 +270,22 @@  class UserProfileOptoutFormTest(TestCase):
         form_re = self._form_re(self.optout_url, self.user.email)
         response = self.client.get(self.url)
         self.assertEquals(response.status_code, 200)
-        self.assertTrue(form_re.search(response.content) is not None)
+        self.assertTrue(form_re.search(response.content.decode()) is not None)
 
     def testMainEmailOptinForm(self):
         EmailOptout(email = self.user.email).save()
         form_re = self._form_re(self.optin_url, self.user.email)
         response = self.client.get(self.url)
         self.assertEquals(response.status_code, 200)
-        self.assertTrue(form_re.search(response.content) is not None)
+        self.assertTrue(form_re.search(response.content.decode()) is not None)
 
     def testSecondaryEmailOptoutForm(self):
         p = Person(email = self.secondary_email, user = self.user)
         p.save()
-        
         form_re = self._form_re(self.optout_url, p.email)
         response = self.client.get(self.url)
         self.assertEquals(response.status_code, 200)
-        self.assertTrue(form_re.search(response.content) is not None)
+        self.assertTrue(form_re.search(response.content.decode()) is not None)
 
     def testSecondaryEmailOptinForm(self):
         p = Person(email = self.secondary_email, user = self.user)
@@ -296,4 +295,4 @@  class UserProfileOptoutFormTest(TestCase):
         form_re = self._form_re(self.optin_url, p.email)
         response = self.client.get(self.url)
         self.assertEquals(response.status_code, 200)
-        self.assertTrue(form_re.search(response.content) is not None)
+        self.assertTrue(form_re.search(response.content.decode()) is not None)
diff --git a/patchwork/tests/test_mboxviews.py b/patchwork/tests/test_mboxviews.py
index 8c98351..37bab74 100644
--- a/patchwork/tests/test_mboxviews.py
+++ b/patchwork/tests/test_mboxviews.py
@@ -173,7 +173,7 @@  class MboxDateHeaderTest(TestCase):
 
     def testDateHeader(self):
         response = self.client.get('/patch/%d/mbox/' % self.patch.id)
-        mail = email.message_from_string(response.content)
+        mail = email.message_from_string(response.content.decode())
         mail_date = dateutil.parser.parse(mail['Date'])
         # patch dates are all in UTC
         patch_date = self.patch.date.replace(tzinfo=dateutil.tz.tzutc(),
@@ -190,7 +190,7 @@  class MboxDateHeaderTest(TestCase):
         self.patch.save()
 
         response = self.client.get('/patch/%d/mbox/' % self.patch.id)
-        mail = email.message_from_string(response.content)
+        mail = email.message_from_string(response.content.decode())
         mail_date = dateutil.parser.parse(mail['Date'])
         self.assertEqual(mail_date, date)
 
diff --git a/patchwork/tests/test_person.py b/patchwork/tests/test_person.py
index ef35da6..9b91115 100644
--- a/patchwork/tests/test_person.py
+++ b/patchwork/tests/test_person.py
@@ -39,14 +39,14 @@  class SubmitterCompletionTest(TestCase):
     def testNameComplete(self):
         response = self.client.get('/submitter/', {'q': 'name'})
         self.assertEquals(response.status_code, 200)
-        data = json.loads(response.content)
+        data = json.loads(response.content.decode())
         self.assertEquals(len(data), 1)
         self.assertEquals(data[0]['name'], 'Test Name')
 
     def testEmailComplete(self):
         response = self.client.get('/submitter/', {'q': 'test2'})
         self.assertEquals(response.status_code, 200)
-        data = json.loads(response.content)
+        data = json.loads(response.content.decode())
         self.assertEquals(len(data), 1)
         self.assertEquals(data[0]['email'], 'test2@example.com')
 
@@ -56,5 +56,5 @@  class SubmitterCompletionTest(TestCase):
             person.save()
         response = self.client.get('/submitter/', {'q': 'test', 'l': 5})
         self.assertEquals(response.status_code, 200)
-        data = json.loads(response.content)
+        data = json.loads(response.content.decode())
         self.assertEquals(len(data), 5)
diff --git a/patchwork/views/patch.py b/patchwork/views/patch.py
index cb017fb..7abb22f 100644
--- a/patchwork/views/patch.py
+++ b/patchwork/views/patch.py
@@ -21,6 +21,7 @@  from __future__ import absolute_import
 
 from django.http import HttpResponse, HttpResponseForbidden
 from django.shortcuts import render_to_response, get_object_or_404
+from django.utils import six
 
 from patchwork.forms import PatchForm, CreateBundleForm
 from patchwork.models import Patch, Project, Bundle
@@ -96,7 +97,11 @@  def content(request, patch_id):
 def mbox(request, patch_id):
     patch = get_object_or_404(Patch, id=patch_id)
     response = HttpResponse(content_type="text/plain")
-    response.write(patch_to_mbox(patch).as_string(True))
+    # NOTE(stephenfin) http://stackoverflow.com/a/28584090/613428
+    if six.PY3:
+        response.write(patch_to_mbox(patch).as_bytes(True).decode())
+    else:
+        response.write(patch_to_mbox(patch).as_string(True))
     response['Content-Disposition'] = 'attachment; filename=' + \
         patch.filename().replace(';', '').replace('\n', '')
     return response