From patchwork Sun Nov 29 22:10:49 2015 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Stephen Finucane X-Patchwork-Id: 549834 Return-Path: X-Original-To: incoming@patchwork.ozlabs.org Delivered-To: patchwork-incoming@bilbo.ozlabs.org Received: from lists.ozlabs.org (lists.ozlabs.org [103.22.144.68]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id 613A91401E7 for ; Mon, 30 Nov 2015 09:12:31 +1100 (AEDT) Received: from lists.ozlabs.org (lists.ozlabs.org [IPv6:2401:3900:2:1::3]) by lists.ozlabs.org (Postfix) with ESMTP id 421F61A0C39 for ; Mon, 30 Nov 2015 09:12:31 +1100 (AEDT) X-Original-To: patchwork@lists.ozlabs.org Delivered-To: patchwork@lists.ozlabs.org Received: from mga11.intel.com (mga11.intel.com [192.55.52.93]) by lists.ozlabs.org (Postfix) with ESMTP id 9404A1A095B for ; Mon, 30 Nov 2015 09:11:14 +1100 (AEDT) Received: from orsmga001.jf.intel.com ([10.7.209.18]) by fmsmga102.fm.intel.com with ESMTP; 29 Nov 2015 14:11:13 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.20,360,1444719600"; d="scan'208";a="830429366" Received: from irvmail001.ir.intel.com ([163.33.26.43]) by orsmga001.jf.intel.com with ESMTP; 29 Nov 2015 14:11:12 -0800 Received: from sivswdev01.ir.intel.com (sivswdev01.ir.intel.com [10.237.217.45]) by irvmail001.ir.intel.com (8.14.3/8.13.6/MailSET/Hub) with ESMTP id tATMBBAp024165; Sun, 29 Nov 2015 22:11:12 GMT Received: from sivswdev01.ir.intel.com (localhost [127.0.0.1]) by sivswdev01.ir.intel.com with ESMTP id tATMBB2Y004913; Sun, 29 Nov 2015 22:11:11 GMT Received: (from sfinucan@localhost) by sivswdev01.ir.intel.com with id tATMBB4x004909; Sun, 29 Nov 2015 22:11:11 GMT From: Stephen Finucane To: patchwork@lists.ozlabs.org Subject: [PATCH 8/9] py3: Resolve unicode issues Date: Sun, 29 Nov 2015 22:10:49 +0000 Message-Id: <1448835050-1572-9-git-send-email-stephen.finucane@intel.com> X-Mailer: git-send-email 2.0.0 In-Reply-To: <1448835050-1572-1-git-send-email-stephen.finucane@intel.com> References: <1448835050-1572-1-git-send-email-stephen.finucane@intel.com> X-BeenThere: patchwork@lists.ozlabs.org X-Mailman-Version: 2.1.20 Precedence: list List-Id: Patchwork development List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Errors-To: patchwork-bounces+incoming=patchwork.ozlabs.org@lists.ozlabs.org Sender: "Patchwork" Python 3 is unicode only. While many of the issues with unicode, such as the now invalid 'u' prefix, have already been resolved, there are a few more issues. Many of these issues are related to HTTPResponse.content, which returns bytes and needs to be "decoded" in order to perform actions like concatenation with str objects (unicode). Where possible, make use of assertContains, per the Django documentation (http://bit.ly/1lRDYie), else fall back to including a 'decode' statement. Signed-off-by: Stephen Finucane --- patchwork/bin/parsemail.py | 4 +++- patchwork/parser.py | 2 +- patchwork/tests/test_bundles.py | 4 ++-- patchwork/tests/test_filters.py | 4 ++-- patchwork/tests/test_list.py | 2 +- patchwork/tests/test_mail_settings.py | 27 +++++++++++++-------------- patchwork/tests/test_mboxviews.py | 4 ++-- patchwork/tests/test_person.py | 6 +++--- patchwork/views/patch.py | 7 ++++++- 9 files changed, 33 insertions(+), 27 deletions(-) diff --git a/patchwork/bin/parsemail.py b/patchwork/bin/parsemail.py index 2bc080c..43d733f 100755 --- a/patchwork/bin/parsemail.py +++ b/patchwork/bin/parsemail.py @@ -68,7 +68,9 @@ def clean_header(header): (frag_str, frag_encoding) = fragment if frag_encoding: return frag_str.decode(frag_encoding) - return frag_str.decode() + elif isinstance(frag_str, six.binary_type): # python 2 + return frag_str.decode() + return frag_str fragments = list(map(decode, decode_header(header))) diff --git a/patchwork/parser.py b/patchwork/parser.py index 84f704a..b65e50c 100644 --- a/patchwork/parser.py +++ b/patchwork/parser.py @@ -230,7 +230,7 @@ def hash_patch(str): # other lines are ignored continue - hash.update(line.encode('utf-8') + '\n') + hash.update((line + '\n').encode('utf-8')) return hash diff --git a/patchwork/tests/test_bundles.py b/patchwork/tests/test_bundles.py index 4b982a5..d5d5d2f 100644 --- a/patchwork/tests/test_bundles.py +++ b/patchwork/tests/test_bundles.py @@ -109,7 +109,7 @@ class BundleViewTest(BundleTestBase): pos = 0 for patch in self.patches: - next_pos = response.content.find(patch.name) + next_pos = response.content.decode().find(patch.name) # ensure that this patch is after the previous self.failUnless(next_pos > pos) pos = next_pos @@ -126,7 +126,7 @@ class BundleViewTest(BundleTestBase): response = self.client.get(bundle_url(self.bundle)) pos = len(response.content) for patch in self.patches: - next_pos = response.content.find(patch.name) + next_pos = response.content.decode().find(patch.name) # ensure that this patch is now *before* the previous self.failUnless(next_pos < pos) pos = next_pos diff --git a/patchwork/tests/test_filters.py b/patchwork/tests/test_filters.py index ae1ff7c..c92146b 100644 --- a/patchwork/tests/test_filters.py +++ b/patchwork/tests/test_filters.py @@ -35,8 +35,8 @@ class FilterQueryStringTest(TestCase): url = '/project/%s/list/?submitter=a%%26b=c' % project.linkname response = self.client.get(url) self.failUnlessEqual(response.status_code, 200) - self.failIf('submitter=a&b=c' in response.content) - self.failIf('submitter=a&b=c' in response.content) + self.assertNotContains(response, 'submitter=a&b=c') + self.assertNotContains(response, 'submitter=a&b=c') def testUTF8QSHandling(self): """test that non-ascii characters can be handled by the filter diff --git a/patchwork/tests/test_list.py b/patchwork/tests/test_list.py index 0f23d13..62ff121 100644 --- a/patchwork/tests/test_list.py +++ b/patchwork/tests/test_list.py @@ -77,7 +77,7 @@ class PatchOrderTest(TestCase): def _extract_patch_ids(self, response): id_re = re.compile('may' in response.content) + self.assertContains(response, 'may') optout_url = reverse('patchwork.views.mail.optout') - self.assertTrue(('action="%s"' % optout_url) in response.content) + self.assertContains(response, ('action="%s"' % optout_url)) def testMailSettingsPOSTOptedOut(self): email = u'foo@example.com' @@ -74,9 +74,9 @@ class MailSettingsTest(TestCase): self.assertEquals(response.status_code, 200) self.assertTemplateUsed(response, 'patchwork/mail-settings.html') self.assertEquals(response.context['is_optout'], True) - self.assertTrue('may not' in response.content) + self.assertContains(response, 'may not') optin_url = reverse('patchwork.views.mail.optin') - self.assertTrue(('action="%s"' % optin_url) in response.content) + self.assertContains(response, ('action="%s"' % optin_url)) class OptoutRequestTest(TestCase): @@ -98,7 +98,7 @@ class OptoutRequestTest(TestCase): # check confirmation page self.assertEquals(response.status_code, 200) self.assertEquals(response.context['confirmation'], conf) - self.assertTrue(email in response.content) + self.assertContains(response, email) # check email url = reverse('patchwork.views.confirm', kwargs = {'key': conf.key}) @@ -140,7 +140,7 @@ class OptoutTest(TestCase): self.assertEquals(response.status_code, 200) self.assertTemplateUsed(response, 'patchwork/optout.html') - self.assertTrue(self.email in response.content) + self.assertContains(response, self.email) # check that we've got an optout in the list self.assertEquals(EmailOptout.objects.count(), 1) @@ -178,7 +178,7 @@ class OptinRequestTest(TestCase): # check confirmation page self.assertEquals(response.status_code, 200) self.assertEquals(response.context['confirmation'], conf) - self.assertTrue(self.email in response.content) + self.assertContains(response, self.email) # check email url = reverse('patchwork.views.confirm', kwargs = {'key': conf.key}) @@ -221,7 +221,7 @@ class OptinTest(TestCase): self.assertEquals(response.status_code, 200) self.assertTemplateUsed(response, 'patchwork/optin.html') - self.assertTrue(self.email in response.content) + self.assertContains(response, self.email) # check that there's no optout remaining self.assertEquals(EmailOptout.objects.count(), 0) @@ -243,7 +243,7 @@ class OptinWithoutOptoutTest(TestCase): # check for an error message self.assertEquals(response.status_code, 200) self.assertTrue(bool(response.context['error'])) - self.assertTrue('not on the patchwork opt-out list' in response.content) + self.assertContains(response, 'not on the patchwork opt-out list') class UserProfileOptoutFormTest(TestCase): """Test that the correct optin/optout forms appear on the user profile @@ -270,23 +270,22 @@ class UserProfileOptoutFormTest(TestCase): form_re = self._form_re(self.optout_url, self.user.email) response = self.client.get(self.url) self.assertEquals(response.status_code, 200) - self.assertTrue(form_re.search(response.content) is not None) + self.assertTrue(form_re.search(response.content.decode()) is not None) def testMainEmailOptinForm(self): EmailOptout(email = self.user.email).save() form_re = self._form_re(self.optin_url, self.user.email) response = self.client.get(self.url) self.assertEquals(response.status_code, 200) - self.assertTrue(form_re.search(response.content) is not None) + self.assertTrue(form_re.search(response.content.decode()) is not None) def testSecondaryEmailOptoutForm(self): p = Person(email = self.secondary_email, user = self.user) p.save() - form_re = self._form_re(self.optout_url, p.email) response = self.client.get(self.url) self.assertEquals(response.status_code, 200) - self.assertTrue(form_re.search(response.content) is not None) + self.assertTrue(form_re.search(response.content.decode()) is not None) def testSecondaryEmailOptinForm(self): p = Person(email = self.secondary_email, user = self.user) @@ -296,4 +295,4 @@ class UserProfileOptoutFormTest(TestCase): form_re = self._form_re(self.optin_url, p.email) response = self.client.get(self.url) self.assertEquals(response.status_code, 200) - self.assertTrue(form_re.search(response.content) is not None) + self.assertTrue(form_re.search(response.content.decode()) is not None) diff --git a/patchwork/tests/test_mboxviews.py b/patchwork/tests/test_mboxviews.py index 8c98351..37bab74 100644 --- a/patchwork/tests/test_mboxviews.py +++ b/patchwork/tests/test_mboxviews.py @@ -173,7 +173,7 @@ class MboxDateHeaderTest(TestCase): def testDateHeader(self): response = self.client.get('/patch/%d/mbox/' % self.patch.id) - mail = email.message_from_string(response.content) + mail = email.message_from_string(response.content.decode()) mail_date = dateutil.parser.parse(mail['Date']) # patch dates are all in UTC patch_date = self.patch.date.replace(tzinfo=dateutil.tz.tzutc(), @@ -190,7 +190,7 @@ class MboxDateHeaderTest(TestCase): self.patch.save() response = self.client.get('/patch/%d/mbox/' % self.patch.id) - mail = email.message_from_string(response.content) + mail = email.message_from_string(response.content.decode()) mail_date = dateutil.parser.parse(mail['Date']) self.assertEqual(mail_date, date) diff --git a/patchwork/tests/test_person.py b/patchwork/tests/test_person.py index ef35da6..9b91115 100644 --- a/patchwork/tests/test_person.py +++ b/patchwork/tests/test_person.py @@ -39,14 +39,14 @@ class SubmitterCompletionTest(TestCase): def testNameComplete(self): response = self.client.get('/submitter/', {'q': 'name'}) self.assertEquals(response.status_code, 200) - data = json.loads(response.content) + data = json.loads(response.content.decode()) self.assertEquals(len(data), 1) self.assertEquals(data[0]['name'], 'Test Name') def testEmailComplete(self): response = self.client.get('/submitter/', {'q': 'test2'}) self.assertEquals(response.status_code, 200) - data = json.loads(response.content) + data = json.loads(response.content.decode()) self.assertEquals(len(data), 1) self.assertEquals(data[0]['email'], 'test2@example.com') @@ -56,5 +56,5 @@ class SubmitterCompletionTest(TestCase): person.save() response = self.client.get('/submitter/', {'q': 'test', 'l': 5}) self.assertEquals(response.status_code, 200) - data = json.loads(response.content) + data = json.loads(response.content.decode()) self.assertEquals(len(data), 5) diff --git a/patchwork/views/patch.py b/patchwork/views/patch.py index cb017fb..7abb22f 100644 --- a/patchwork/views/patch.py +++ b/patchwork/views/patch.py @@ -21,6 +21,7 @@ from __future__ import absolute_import from django.http import HttpResponse, HttpResponseForbidden from django.shortcuts import render_to_response, get_object_or_404 +from django.utils import six from patchwork.forms import PatchForm, CreateBundleForm from patchwork.models import Patch, Project, Bundle @@ -96,7 +97,11 @@ def content(request, patch_id): def mbox(request, patch_id): patch = get_object_or_404(Patch, id=patch_id) response = HttpResponse(content_type="text/plain") - response.write(patch_to_mbox(patch).as_string(True)) + # NOTE(stephenfin) http://stackoverflow.com/a/28584090/613428 + if six.PY3: + response.write(patch_to_mbox(patch).as_bytes(True).decode()) + else: + response.write(patch_to_mbox(patch).as_string(True)) response['Content-Disposition'] = 'attachment; filename=' + \ patch.filename().replace(';', '').replace('\n', '') return response