Ticket #17375: issue_17375.2.diff

django/core/management/commands/makemessages.py

@@ -3 +3 @@
 import os
 import re
 import sys
+import tempfile
 from itertools import dropwhile
 from optparse import make_option
 from subprocess import PIPE, Popen
@@ -13 +14 @@
 
 plural_forms_re = re.compile(r'^(?P<value>"Plural-Forms.+?\\n")\s*$', re.MULTILINE | re.DOTALL)
 
+
 def handle_extensions(extensions=('html',), ignored=('py',)):
     """
     Organizes multiple extensions that are separated with commas or passed by
@@ -37 +39 @@
             ext_list[i] = '.%s' % ext_list[i]
     return set([x for x in ext_list if x.strip('.') not in ignored])
 
+
 def _popen(cmd):
     """
     Friendly wrapper around Popen for Windows
@@ -44 +47 @@
     p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE, close_fds=os.name != 'nt', universal_newlines=True)
     return p.communicate()
 
+
 def walk(root, topdown=True, onerror=None, followlinks=False):
     """
     A version of os.walk that can follow symlinks for Python < 2.6
@@ -57 +61 @@
                     for link_dirpath, link_dirnames, link_filenames in walk(p):
                         yield (link_dirpath, link_dirnames, link_filenames)
 
+
 def is_ignored(path, ignore_patterns):
     """
     Helper function to check if the given path should be ignored or not.
@@ -66 +71 @@
             return True
     return False
 
+
 def find_files(root, ignore_patterns, verbosity, symlinks=False):
     """
     Helper function to get all files in the given root.
@@ -82 +88 @@
     all_files.sort()
     return all_files
 
+
 def copy_plural_forms(msgs, locale, domain, verbosity):
     """
     Copies plural forms header contents from a Django catalog of locale to
@@ -112 +119 @@
                 break
     return msgs
 
-def write_pot_file(potfile, msgs, file, work_file, is_templatized):
+
+def write_pot_file(potfile, msgs, all_files):
     """
     Write the :param potfile: POT file with the :param msgs: contents,
     previously making sure its format is valid.
     """
-    if is_templatized:
-        old = '#: ' + work_file[2:]
-        new = '#: ' + file[2:]
-        msgs = msgs.replace(old, new)
+    for f_data in all_files:
+        if f_data['is_templatized']:
+            old = '#: ' + f_data['work'][2:]
+            new = '#: ' + f_data['orig'][2:]
+            msgs = msgs.replace(old, new)
     if os.path.exists(potfile):
         # Strip the header
         msgs = '\n'.join(dropwhile(len, msgs.split('\n')))
@@ -132 +141 @@
     finally:
         f.close()
 
-def process_file(file, dirpath, potfile, domain, verbosity, extensions, wrap,
-        location):
-    """
-    Extract translatable literals from :param file: for :param domain:
-    creating or updating the :param potfile: POT file.
-
-    Uses the xgettext GNU gettext utility.
-    """
-
-    from django.utils.translation import templatize
-
-    if verbosity > 1:
-        sys.stdout.write('processing file %s in %s\n' % (file, dirpath))
-    _, file_ext = os.path.splitext(file)
-    if domain == 'djangojs' and file_ext in extensions:
-        is_templatized = True
-        orig_file = os.path.join(dirpath, file)
-        src_data = open(orig_file).read()
-        src_data = prepare_js_for_gettext(src_data)
-        thefile = '%s.c' % file
-        work_file = os.path.join(dirpath, thefile)
-        f = open(work_file, "w")
-        try:
-            f.write(src_data)
-        finally:
-            f.close()
-        cmd = (
-            'xgettext -d %s -L C %s %s --keyword=gettext_noop '
-            '--keyword=gettext_lazy --keyword=ngettext_lazy:1,2 '
-            '--keyword=pgettext:1c,2 --keyword=npgettext:1c,2,3 '
-            '--from-code UTF-8 --add-comments=Translators -o - "%s"' % (
-                domain, wrap, location, work_file
-            )
-        )
-    elif domain == 'django' and (file_ext == '.py' or file_ext in extensions):
-        thefile = file
-        orig_file = os.path.join(dirpath, file)
-        is_templatized = file_ext in extensions
-        if is_templatized:
-            src_data = open(orig_file, "rU").read()
-            thefile = '%s.py' % file
-            content = templatize(src_data, orig_file[2:])
-            f = open(os.path.join(dirpath, thefile), "w")
-            try:
-                f.write(content)
-            finally:
-                f.close()
-        work_file = os.path.join(dirpath, thefile)
-        cmd = (
-            'xgettext -d %s -L Python %s %s --keyword=gettext_noop '
-            '--keyword=gettext_lazy --keyword=ngettext_lazy:1,2 '
-            '--keyword=ugettext_noop --keyword=ugettext_lazy '
-            '--keyword=ungettext_lazy:1,2 --keyword=pgettext:1c,2 '
-            '--keyword=npgettext:1c,2,3 --keyword=pgettext_lazy:1c,2 '
-            '--keyword=npgettext_lazy:1c,2,3 --from-code UTF-8 '
-            '--add-comments=Translators -o - "%s"' % (
-                domain, wrap, location, work_file)
-        )
-    else:
-        return
-    msgs, errors = _popen(cmd)
-    if errors:
-        if is_templatized:
-            os.unlink(work_file)
-        if os.path.exists(potfile):
-            os.unlink(potfile)
-        raise CommandError(
-            "errors happened while running xgettext on %s\n%s" %
-            (file, errors))
-    if msgs:
-        write_pot_file(potfile, msgs, orig_file, work_file, is_templatized)
-    if is_templatized:
-        os.unlink(work_file)
 
 def write_po_file(pofile, potfile, domain, locale, verbosity,
         copy_pforms, wrap, location, no_obsolete):
@@ -248 +184 @@
             raise CommandError(
                 "errors happened while running msgattrib\n%s" % errors)
 
+
 def make_messages(locale=None, domain='django', verbosity=1, all=False,
         extensions=None, symlinks=False, ignore_patterns=None, no_wrap=False,
         no_location=False, no_obsolete=False):
@@ -323 +260 @@
         if os.path.exists(potfile):
             os.unlink(potfile)
 
-        for dirpath, file in find_files(".", ignore_patterns, verbosity,
-                symlinks=symlinks):
-            process_file(file, dirpath, potfile, domain, verbosity, extensions,
-                    wrap, location)
+        process_files(potfile, domain, verbosity, extensions,
+                      wrap, location, ignore_patterns, symlinks)
 
         if os.path.exists(potfile):
             write_po_file(pofile, potfile, domain, locale, verbosity,
                     not invoked_for_django, wrap, location, no_obsolete)
 
 
+def process_files(potfile, domain, verbosity, extensions, wrap, location,
+                  ignore_patterns, symlinks):
+    """
+    Extract translatable literals from current directory for :param domain:
+    creating or updating the :param potfile: POT file.
+
+    Uses the xgettext GNU gettext utility.
+    """
+    from django.utils.translation import templatize
+
+    # stores information about all processed files
+    all_files = []
+
+    # prepare command to run xgettext depending on domain type
+    if domain == 'djangojs':
+        cmd = (
+            'xgettext -d %s -L C %s %s --keyword=gettext_noop '
+            '--keyword=gettext_lazy --keyword=ngettext_lazy:1,2 '
+            '--keyword=pgettext:1c,2 --keyword=npgettext:1c,2,3 '
+            '--from-code UTF-8 --add-comments=Translators -o - -f %s'
+        )
+    elif domain == 'django':
+        cmd = (
+            'xgettext -d %s -L Python %s %s --keyword=gettext_noop '
+            '--keyword=gettext_lazy --keyword=ngettext_lazy:1,2 '
+            '--keyword=ugettext_noop --keyword=ugettext_lazy '
+            '--keyword=ungettext_lazy:1,2 --keyword=pgettext:1c,2 '
+            '--keyword=npgettext:1c,2,3 --keyword=pgettext_lazy:1c,2 '
+            '--keyword=npgettext_lazy:1c,2,3 --from-code UTF-8 '
+            '--add-comments=Translators -o - -f %s'
+        )
+
+    for dirpath, file in find_files(".", ignore_patterns, verbosity,
+                                    symlinks=symlinks):
+        if verbosity > 1:
+            sys.stdout.write('processing file %s in %s\n' % (file, dirpath))
+        _, file_ext = os.path.splitext(file)
+
+        is_templatized = file_ext in extensions
+
+        # path to the processed file
+        orig_file = os.path.join(dirpath, file)
+
+        # store information about processed file;
+        # 'orig' - path to original file
+        # 'work' - path to either original file or templatized version if one is created
+        # 'is_templatized' - defines if file was templatized
+        file_dict = {'orig': orig_file, 'work': orig_file, 'is_templatized': is_templatized}
+
+        # not really necessary to initialize these here but added for clarity
+        thefile = ''
+        content = ''
+
+        if domain == 'djangojs' and is_templatized:
+            src_data = open(orig_file).read()
+            content = prepare_js_for_gettext(src_data)
+            thefile = '%s.c' % file
+        elif domain == 'django' and (file_ext == '.py' or is_templatized):
+            if is_templatized:
+                src_data = open(orig_file, "rU").read()
+                content = templatize(src_data, orig_file[2:])
+                thefile = '%s.py' % file
+        else:
+            continue
+
+        if is_templatized:
+            # for templatized ones create work file
+            work_file = os.path.join(dirpath, thefile)
+            file_dict['work'] = work_file
+
+            f = open(work_file, "w")
+            try:
+                f.write(content)
+            finally:
+                f.close()
+
+        all_files.append(file_dict)
+
+    if all_files:
+        # prepare command that runs xgettext with path to file that
+        # contains paths of all files to process
+        input_files = '\n'.join([f_data['work'] for f_data in all_files])
+
+        # create temporary file for xgettext
+        f_handle, input_files_path = tempfile.mkstemp()
+        f = open(input_files_path, 'w')
+        try:
+            f.write(input_files)
+        finally:
+            f.close()
+
+        cmd = cmd % (domain, wrap, location, input_files_path)
+
+        msgs, errors = _popen(cmd)
+
+        os.unlink(input_files_path)
+
+        if errors:
+            # check if file exists because when symlinks are used it might
+            # happen that it was already removed
+            [os.unlink(f_data['work']) for f_data in all_files
+               if f_data['is_templatized'] and os.path.exists(f_data['work'])]
+
+            input_files = ', '.join([f_data['work'] for f_data in all_files])
+            raise CommandError(
+                "errors happened while running xgettext on %s\n%s" %
+                (input_files, errors))
+        if msgs:
+            write_pot_file(potfile, msgs, all_files)
+        # check if file exists because when symlinks are used it might
+        # happen that it was already removed
+        [os.unlink(f_data['work']) for f_data in all_files
+               if f_data['is_templatized'] and os.path.exists(f_data['work'])]
+
+
 class Command(NoArgsCommand):
     option_list = NoArgsCommand.option_list + (
         make_option('--locale', '-l', default=None, dest='locale',

tests/regressiontests/i18n/commands/extraction.py

@@ -41 +41 @@
         msgid = re.escape(msgid)
         return self.assertTrue(re.search('^msgid %s' % msgid, s, re.MULTILINE), 'Could not find %(q)s%(n)s%(q)s in generated PO file' % {'n':needle, 'q':q})
 
+    def assertMsgIdPlural(self, msgid, s, use_quotes=True):
+        q = '"'
+        if use_quotes:
+            msgid = '"%s"' % msgid
+            q = "'"
+        needle = 'msgid_plural %s' % msgid
+        msgid = re.escape(msgid)
+        return self.assertTrue(re.search('^msgid_plural %s' % msgid, s, re.MULTILINE), 'Could not find %(q)s%(n)s%(q)s in generated PO file' % {'n':needle, 'q':q})
+
     def assertNotMsgId(self, msgid, s, use_quotes=True):
         if use_quotes:
             msgid = '"%s"' % msgid
@@ -263 +272 @@
         with open(self.PO_FILE, 'r') as fp:
             po_contents = fp.read()
             self.assertTrue('#: templates/test.html:55' in po_contents)
+
+
+class MultipleFilesPluralExtractorTests(ExtractorTests):
+
+    def setUp(self):
+        self._cwd = os.getcwd()
+        self.test_dir = os.path.abspath(os.path.dirname(__file__))
+        self.second_test_template = os.path.join(self.test_dir, 'templates', 'plural_test_template.html')
+
+        second_test_template = '''{% load i18n %}{% trans 'My string' %}'''
+
+        if not os.path.exists(self.second_test_template):
+            f = open(self.second_test_template, 'w+')
+            try:
+                f.write(second_test_template)
+            finally:
+                f.close()
+
+    def tearDown(self):
+        super(MultipleFilesPluralExtractorTests, self).tearDown()
+        os.chdir(self.test_dir)
+        try:
+            os.remove(self.second_test_template)
+        except OSError:
+            pass
+        os.chdir(self._cwd)
+
+    def test_multiple_files_plural_extraction(self):
+        os.chdir(self.test_dir)
+        management.call_command('makemessages', locale=LOCALE, verbosity=0, symlinks=True)
+        self.assertTrue(os.path.exists(self.PO_FILE))
+        with open(self.PO_FILE, 'r') as fp:
+            po_contents = fp.read()
+            self.assertMsgId('My string', po_contents)
+            self.assertMsgIdPlural('My strings', po_contents)

tests/regressiontests/i18n/commands/templates/test.html

@@ -77 +77 @@
 {% trans "Shouldn't double escape this sequence %% either" context "ctx1" %}
 {% trans "Looks like a str fmt spec %s but shouldn't be interpreted as such" %}
 {% trans "Looks like a str fmt spec % o but shouldn't be interpreted as such" %}
+{% blocktrans count counter=mylist|length %}My string{% plural %}My strings{% endblocktrans %}

tests/regressiontests/i18n/tests.py

@@ -29 +29 @@
     from .commands.extraction import (ExtractorTests, BasicExtractorTests,
         JavascriptExtractorTests, IgnoredExtractorTests, SymlinkExtractorTests,
         CopyPluralFormsExtractorTests, NoWrapExtractorTests,
-        NoLocationExtractorTests)
+        NoLocationExtractorTests, MultipleFilesPluralExtractorTests)
 if can_run_compilation_tests:
     from .commands.compilation import (PoFileTests, PoFileContentsTests,
         PercentRenderingTests)