Ticket #16052: mysql_utf8bin_unicode.diff

tests/regressiontests/backends/tests.py

@@ -1 +1 @@
 # -*- coding: utf-8 -*-
 # Unit and doctests for specific database backends.
 import datetime
+import sys
 
 from django.conf import settings
 from django.core.management.color import no_style
@@ -322 +323 @@
             a.save()
         except IntegrityError:
             pass
+
+class MySQLEncodingTest(TestCase):
+
+    @unittest.skipUnless(connection.vendor == 'mysql',
+                         "MySQL specific utf8_bin test.")
+    def test_utfbin_unicode(self):
+        """
+        Ensure CharField is unicode with both utf_bin and default encoding.
+        """
+
+        models.Person.objects.create(first_name='Django', last_name='Reinhardt')
+
+        cur = connections['default'].cursor()
+        cur.execute('ALTER TABLE backends_person ' +
+                    'MODIFY first_name VARCHAR(20) ' +
+                    'CHARACTER SET utf8 COLLATE utf8_bin;')
+        cur.close()
+
+        person = models.Person.objects.get(last_name='Reinhardt')
+
+        self.assertIsInstance(person.first_name, unicode)   # utf_bin
+        self.assertIsInstance(person.last_name, unicode)    # default
+
+    @unittest.skipUnless(connection.vendor == 'mysql',
+                         "MySQL specific utf8_bin test.")
+    def test_utfbin_unicode_needed(self):
+        """
+        Ensure CharField is unicode with both utf_bin and default encoding,
+        with an actual non-ascii string.
+        """
+
+        models.Person.objects.create(first_name='汉语/漢語', last_name='Reinhardt')
+
+        cur = connections['default'].cursor()
+        cur.execute('ALTER TABLE backends_person ' +
+                    'MODIFY first_name VARCHAR(20) ' +
+                    'CHARACTER SET utf8 COLLATE utf8_bin;')
+        cur.close()
+
+        person = models.Person.objects.get(last_name='Reinhardt')
+
+        self.assertIsInstance(person.first_name, unicode)   # utf_bin
+        self.assertIsInstance(person.last_name, unicode)    # default
+
+    @unittest.skipUnless(connection.vendor == 'mysql',
+                         "MySQL specific conversion test.")
+    def test_convert_var_string_unicode(self):
+        """Test the convert_var_string method with unicode data"""
+        from django.db.backends.mysql.base import convert_var_string
+
+        result = convert_var_string('汉语/漢語')
+        self.assertIsInstance(result, unicode)
+
+    @unittest.skipUnless(connection.vendor == 'mysql',
+                         "MySQL specific conversion test.")
+    def test_convert_var_string_binary(self):
+        """Test the convert_var_string method with binary data"""
+        from django.db.backends.mysql.base import convert_var_string
+
+        # Use the python executable as source of binary data
+        data = open(sys.executable, 'rb').read()
+
+        result = convert_var_string(data)
+        self.assertIsInstance(result, str)
+

django/db/backends/mysql/base.py

@@ -33 +33 @@
 from django.db.backends.mysql.introspection import DatabaseIntrospection
 from django.db.backends.mysql.validation import DatabaseValidation
 from django.utils.safestring import SafeString, SafeUnicode
+from django.utils.encoding import force_unicode
 
 # Raise exceptions for database warnings if DEBUG is on
 from django.conf import settings
@@ -43 +44 @@
 DatabaseError = Database.DatabaseError
 IntegrityError = Database.IntegrityError
 
+def convert_var_string(data):
+    """
+    Guess whether data is binary or text, and return:
+     - str for binary and UTF-16
+     - unicode for UTF-8, ascii and all other encodings.
+    """
+    if '\0' in data:
+        return str(data)
+    else:
+        return force_unicode(data)
+
 # MySQLdb-1.2.1 returns TIME columns as timedelta -- they are more like
 # timedelta in terms of actual behavior as they are signed and include days --
 # and Django expects time, so we still need to override that. We also need to
@@ -53 +65 @@
     FIELD_TYPE.TIME: util.typecast_time,
     FIELD_TYPE.DECIMAL: util.typecast_decimal,
     FIELD_TYPE.NEWDECIMAL: util.typecast_decimal,
+    FIELD_TYPE.VAR_STRING: [(None, convert_var_string)],
 })
 
 # This should match the numerical portion of the version numbers (we can treat