Ticket #6073: 6073-dont-allow-BOM-in-po-files-1.diff

django/core/management/commands/compilemessages.py

@@ +1 @@
+import codecs
 import os
 import sys
 from optparse import make_option
@@ -8 +9 @@
 except NameError:
     from sets import Set as set     # For Python 2.3
 
+def has_bom(fn):
+    f = open(fn, 'r')
+    sample = f.read(4)
+    return sample[:3] == '\xef\xbb\xbf' or \
+            sample.startswith(codecs.BOM_UTF16_LE) or \
+            sample.startswith(codecs.BOM_UTF16_BE)
+
 def compile_messages(locale=None):
     basedirs = [os.path.join('conf', 'locale'), 'locale']
     if os.environ.get('DJANGO_SETTINGS_MODULE'):
@@ -27 +35 @@
             for f in filenames:
                 if f.endswith('.po'):
                     sys.stderr.write('processing file %s in %s\n' % (f, dirpath))
-                    pf = os.path.splitext(os.path.join(dirpath, f))[0]
+                    fn = os.path.join(dirpath, f)
+                    if has_bom(fn):
+                        raise CommandError("The %s file has a BOM (Byte Order Mark). Django only supports .po files encoded in UTF-8 and without any BOM." % fn)
+                    pf = os.path.splitext(fn)[0]
                     # Store the names of the .mo and .po files in an environment
                     # variable, rather than doing a string replacement into the
                     # command, so that we can take advantage of shell quoting, to

docs/topics/i18n.txt

@@ -650 +650 @@
    ``django-admin compilemessages`` works see `gettext on Windows`_ for more
    information.
 
+.. admonition:: .po files: Encoding and BOM usage.
+
+   Django only supports ``.po`` files encoded in UTF-8 and without any BOM
+   (Byte Order Mark) so if your text editor adds BOM to the beginning of
+   files by default you need to configure it to not do so.
+
 .. _how-django-discovers-language-preference:
 
 3. How Django discovers language preference