Ticket #11838: ticket11838-2.diff

django/core/management/commands/loaddata.py

@@ -6 +6 @@
 
 from django.core.management.base import BaseCommand
 from django.core.management.color import no_style
+from django.core import serializers
+from django.db import transaction
 
 try:
     set
@@ -24 +26 @@
 
     def handle(self, *fixture_labels, **options):
         from django.db.models import get_apps
-        from django.core import serializers
-        from django.db import connection, transaction
+        from django.db import connection
         from django.conf import settings
 
         self.style = no_style()
 
-        verbosity = int(options.get('verbosity', 1))
-        show_traceback = options.get('traceback', False)
+        self.verbosity = int(options.get('verbosity', 1))
+        self.show_traceback = options.get('traceback', False)
 
         # commit is a stealth option - it isn't really useful as
         # a command line option, but it can be useful when invoking
@@ -43 +44 @@
 
         # Keep a count of the installed objects and fixtures
         fixture_count = 0
-        object_count = 0
-        models = set()
+        self.object_count = 0
+        self.models = set()
 
-        humanize = lambda dirname: dirname and "'%s'" % dirname or 'absolute path'
+        self.humanize = lambda dirname: dirname and "'%s'" % dirname or 'absolute path'
 
         # Get a cursor (even though we don't need one yet). This has
         # the side effect of initializing the test database (if
@@ -88 +89 @@
 
             if len(parts) == 1:
                 fixture_name = parts[0]
+                fixture_format = None
                 formats = serializers.get_public_serializer_formats()
             else:
                 fixture_name, format = '.'.join(parts[:-1]), parts[-1]
                 if format in serializers.get_public_serializer_formats():
+                    fixture_format = format
                     formats = [format]
                 else:
                     formats = []
 
             if formats:
-                if verbosity > 1:
+                if self.verbosity > 1:
                     print "Loading '%s' fixtures..." % fixture_name
             else:
                 sys.stderr.write(
@@ -113 +116 @@
                 fixture_dirs = app_fixtures + list(settings.FIXTURE_DIRS) + ['']
 
             for fixture_dir in fixture_dirs:
-                if verbosity > 1:
-                    print "Checking %s for fixtures..." % humanize(fixture_dir)
+                if self.verbosity > 1:
+                    print "Checking %s for fixtures..." % self.humanize(fixture_dir)
 
                 label_found = False
                 for format in formats:
@@ -125 +128 @@
                         else: 
                             file_name = '.'.join([fixture_name, format])
                     
-                        if verbosity > 1:
+                        if self.verbosity > 1:
                             print "Trying %s for %s fixture '%s'..." % \
-                                (humanize(fixture_dir), file_name, fixture_name)
+                                (self.humanize(fixture_dir), file_name, fixture_name)
                         full_path = os.path.join(fixture_dir, file_name)
                         open_method = compression_types[compression_format]                                
                         try: 
@@ -135 +138 @@
                             if label_found:
                                 fixture.close()
                                 print self.style.ERROR("Multiple fixtures named '%s' in %s. Aborting." %
-                                    (fixture_name, humanize(fixture_dir)))
+                                    (fixture_name, self.humanize(fixture_dir)))
                                 transaction.rollback()
                                 transaction.leave_transaction_management()
                                 return
                             else:
                                 fixture_count += 1
-                                objects_in_fixture = 0
-                                if verbosity > 0:
-                                    print "Installing %s fixture '%s' from %s." % \
-                                        (format, fixture_name, humanize(fixture_dir))
-                                try:
-                                    objects = serializers.deserialize(format, fixture)
-                                    for obj in objects:
-                                        objects_in_fixture += 1
-                                        models.add(obj.object.__class__)
-                                        obj.save()
-                                    object_count += objects_in_fixture
-                                    label_found = True
-                                except (SystemExit, KeyboardInterrupt):
-                                    raise
-                                except Exception:
-                                    import traceback
-                                    fixture.close()
-                                    transaction.rollback()
-                                    transaction.leave_transaction_management()
-                                    if show_traceback:
-                                        traceback.print_exc()
-                                    else:
-                                        sys.stderr.write(
-                                            self.style.ERROR("Problem installing fixture '%s': %s\n" %
-                                                 (full_path, ''.join(traceback.format_exception(sys.exc_type, 
-                                                     sys.exc_value, sys.exc_traceback))))) 
+                                label_found, error = self.install_fixture(format, fixture, fixture_name, fixture_dir, full_path)
+                                if error:
                                     return
-                                fixture.close()
-
-                                # If the fixture we loaded contains 0 objects, assume that an
-                                # error was encountered during fixture loading.
-                                if objects_in_fixture == 0:
-                                    sys.stderr.write(
-                                        self.style.ERROR("No fixture data found for '%s'. (File format may be invalid.)" %
-                                            (fixture_name)))
-                                    transaction.rollback()
-                                    transaction.leave_transaction_management()
-                                    return
-
                         except Exception, e:
-                            if verbosity > 1:
+                            if self.verbosity > 1:
                                 print "No %s fixture '%s' in %s." % \
-                                    (format, fixture_name, humanize(fixture_dir))
+                                    (format, fixture_name, self.humanize(fixture_dir))
 
+                # Check for a group of fixtures with the requested name
+                if not fixture_format:
+                    group_dir = os.path.join(fixture_dir, fixture_name)
+                    if os.path.isdir(group_dir):
+                        if label_found:
+                            print self.style.ERROR("A fixture named '%s' and a fixture group named '%s' were both found in %s. Aborting." %
+                                (fixture_name, fixture_name, self.humanize(fixture_dir)))
+                            transaction.rollback()
+                            transaction.leave_transaction_management()
+                            return
+                        else:
+                            labels_found = []
+                            if self.verbosity > 1:
+                                print "Checking %s for a group of fixtures." % \
+                                    self.humanize(group_dir)
+                            for file_name in os.listdir(group_dir):
+                                parts = file_name.split('.')
+                    
+                                if len(parts) > 2 and parts[-1] in compression_types:
+                                    compression_format = parts[-1]
+                                    parts = parts[:-1]
+                                else:
+                                    compression_format = None
+                    
+                                if len(parts) == 2:
+                                    group_fixture_name, format = '.'.join(parts[:-1]), parts[-1]
+                        
+                                    if format in serializers.get_public_serializer_formats():
+                                        full_path = os.path.join(group_dir, file_name)
+                                        if self.verbosity > 1:
+                                            print "Trying %s for %s fixture '%s'..." % \
+                                                (self.humanize(group_dir), file_name,
+                                                 fixture_name)
+                                        open_method = compression_types[compression_format]
+                                        try:
+                                            fixture = open_method(full_path, 'r')
+                                            if group_fixture_name in labels_found:
+                                                fixture.close()
+                                                print self.style.ERROR("Multiple fixtures named '%s' in %s. Aborting." %
+                                                    (group_fixture_name, self.humanize(group_dir)))
+                                                transaction.rollback()
+                                                transaction.leave_transaction_management()
+                                                return
+                                            fixture_count += 1
+                                            label_found, error = self.install_fixture(format, fixture, group_fixture_name, group_dir, full_path)
+                                            if error:
+                                                return
+                                            if label_found:
+                                                labels_found += [label_found]
+                                        except Exception, e:
+                                            if self.verbosity > 1:
+                                                print "No %s fixture '%s' in %s." % \
+                                                    (format, group_fixture_name, self.humanize(group_dir))
+
         # If we found even one object in a fixture, we need to reset the
         # database sequences.
-        if object_count > 0:
-            sequence_sql = connection.ops.sequence_reset_sql(self.style, models)
+        if self.object_count > 0:
+            sequence_sql = connection.ops.sequence_reset_sql(self.style, self.models)
             if sequence_sql:
-                if verbosity > 1:
+                if self.verbosity > 1:
                     print "Resetting sequences"
                 for line in sequence_sql:
                     cursor.execute(line)
@@ -199 +220 @@
             transaction.commit()
             transaction.leave_transaction_management()
 
-        if object_count == 0:
-            if verbosity > 1:
+        if self.object_count == 0:
+            if self.verbosity > 1:
                 print "No fixtures found."
         else:
-            if verbosity > 0:
-                print "Installed %d object(s) from %d fixture(s)" % (object_count, fixture_count)
+            if self.verbosity > 0:
+                print "Installed %d object(s) from %d fixture(s)" % (self.object_count, fixture_count)
 
         # Close the DB connection. This is required as a workaround for an
         # edge case in MySQL: if the same connection is used to
@@ -212 +233 @@
         # incorrect results. See Django #7572, MySQL #37735.
         if commit:
             connection.close()
+
+    def install_fixture(self, format, fixture, fixture_name, fixture_dir, full_path):
+        """
+        Installs the requested fixture.
+        
+        Returns:
+            label_found - The name of the label found.
+            error - Whether there was an error while installing the fixture
+        """
+        error = False
+        label_found = False
+        objects_in_fixture = 0
+        if self.verbosity > 0:
+            print "Installing %s fixture '%s' from %s." % \
+                (format, fixture_name, self.humanize(fixture_dir))
+        try:
+            objects = serializers.deserialize(format, fixture)
+            for obj in objects:
+                objects_in_fixture += 1
+                self.models.add(obj.object.__class__)
+                obj.save()
+            self.object_count += objects_in_fixture
+            label_found = fixture_name
+        except (SystemExit, KeyboardInterrupt):
+            raise
+        except Exception:
+            import traceback
+            fixture.close()
+            transaction.rollback()
+            transaction.leave_transaction_management()
+            if self.show_traceback:
+                traceback.print_exc()
+            else:
+                sys.stderr.write(
+                    self.style.ERROR("Problem installing fixture '%s': %s\n" %
+                         (full_path, ''.join(traceback.format_exception(sys.exc_type, 
+                             sys.exc_value, sys.exc_traceback))))) 
+            error = True
+            return label_found, error
+        fixture.close()
+
+        # If the fixture we loaded contains 0 objects, assume that an
+        # error was encountered during fixture loading.
+        if objects_in_fixture == 0:
+            sys.stderr.write(
+                self.style.ERROR("No fixture data found for '%s'. (File format may be invalid.)" %
+                    (fixture_name)))
+            transaction.rollback()
+            transaction.leave_transaction_management()
+            error = True
+            return label_found, error
+
+        return label_found, error

tests/modeltests/fixtures/fixtures/group3/fixture1.json

@@ +1 @@
+[
+    {
+        "pk": "8", 
+        "model": "fixtures.article", 
+        "fields": {
+            "headline": "Researchers use Python to produce the Ultimate Question of Life, the Universe, and Everything", 
+            "pub_date": "2009-12-06 11:00:00"
+        }
+    }
+]
+ No newline at end of file

tests/modeltests/fixtures/fixtures/group3/fixture2.json

@@ +1 @@
+[
+    {
+        "pk": 8,
+        "model": "fixtures.category",
+        "fields": {
+            "description": "Latest catastrophic disasters",
+            "title": "Catastrophic Disasters"
+        }
+    }
+]
+ No newline at end of file

tests/modeltests/fixtures/fixtures/group4/fixture2.json

@@ +1 @@
+[
+    {
+        "pk": 9,
+        "model": "fixtures.category",
+        "fields": {
+            "description": "Amusing hypochondriac anecdotes",
+            "title": "Hypochondriac Anecdotes"
+        }
+    }
+]
+ No newline at end of file

tests/modeltests/fixtures/fixtures/group1.json

@@ +1 @@
+[
+    {
+        "pk": "6", 
+        "model": "fixtures.article", 
+        "fields": {
+            "headline": "Guido awarded Nobel Prize", 
+            "pub_date": "2009-12-06 11:00:00"
+        }
+    }
+]
+ No newline at end of file

tests/modeltests/fixtures/fixtures/group1/fixture1.json

@@ +1 @@
+[
+    {
+        "pk": "6", 
+        "model": "fixtures.article", 
+        "fields": {
+            "headline": "This fixture should never be installed", 
+            "pub_date": "2009-12-06 11:00:00"
+        }
+    }
+]
+ No newline at end of file

tests/modeltests/fixtures/fixtures/group2/fixture1.json

@@ +1 @@
+[
+    {
+        "pk": "7", 
+        "model": "fixtures.article", 
+        "fields": {
+            "headline": "This fixture will cause the transaction to be rolled back", 
+            "pub_date": "2009-12-06 11:00:00"
+        }
+    }
+]
+ No newline at end of file

tests/modeltests/fixtures/fixtures/group2/fixture1.xml

@@ +1 @@
+<?xml version="1.0" encoding="utf-8"?>
+<django-objects version="1.0">
+    <object pk="7" model="fixtures.article">
+        <field type="CharField" name="headline">This fixture will load, but then be rolled back</field>
+        <field type="DateTimeField" name="pub_date">2009-12-06 11:00:00</field>
+    </object>
+</django-objects>
+ No newline at end of file

tests/modeltests/fixtures/models.py

@@ -159 +159 @@
 >>> management.call_command('loaddata', 'fixture5', verbosity=0) # doctest: +ELLIPSIS
 Multiple fixtures named 'fixture5' in '...fixtures'. Aborting.
 
+# Try to load fixture group 1 using format discovery; this will fail because
+# there is a fixture and a group both named 'group1' in the fixtures
+# directory.
+>>> management.call_command('loaddata', 'group1', verbosity=0) # doctest: +ELLIPSIS
+A fixture named 'group1' and a fixture group named 'group1' were both found in '...fixtures'. Aborting.
+
+# Load the individual fixture group1 with the full filename; this will succeed
+# because the format was explicitly specified. The group of fixtures
+# will be ignored.
+>>> management.call_command('loaddata', 'group1.json', verbosity=0)
+>>> Article.objects.all()
+[<Article: Guido awarded Nobel Prize>, <Article: Python program becomes self aware>]
+
+>>> management.call_command('flush', verbosity=0, interactive=False)
+
+# Try to load group 2; this will fail because there are two fixtures in the
+# group named 'fixture1'.
+>>> management.call_command('loaddata', 'group2', verbosity=0) # doctest: +ELLIPSIS
+Multiple fixtures named 'fixture1' in '.../fixtures/group2'. Aborting.
+
+# Load group 3, which will successfully load 2 fixtures.
+>>> management.call_command('loaddata', 'group3', verbosity=0)
+>>> Article.objects.all()
+[<Article: Researchers use Python to produce the Ultimate Question of Life, the Universe, and Everything>, <Article: Python program becomes self aware>]
+>>> Category.objects.all()
+[<Category: Catastrophic Disasters>]
+
+>>> management.call_command('flush', verbosity=0, interactive=False)
+
+# Load group 4, which includes 1 gzipped fixture and 1 plain text fixture.
+>>> management.call_command('loaddata', 'group4', verbosity=0)
+>>> Article.objects.all()
+[<Article: Pinky successfully ponders what Brain is pondering, lab in uproar>, <Article: Python program becomes self aware>]
+>>> Category.objects.all()
+[<Category: Hypochondriac Anecdotes>]
 """
 
 from django.test import TestCase

AUTHORS

@@ -250 +250 @@
     Bruce Kroeze <http://coderseye.com/>
     krzysiek.pawlik@silvermedia.pl
     Joseph Kocherhans
+    Brandon Konkle (bkonkle) <http://brandonkonkle.com/>
     konrad@gwu.edu
     knox <christobzr@gmail.com>
     David Krauth

docs/howto/initial-data.txt

@@ -82 +82 @@
 but be careful: remember that the data will be refreshed *every time* you run
 :djadmin:`syncdb`. So don't use ``initial_data`` for data you'll want to edit.
 
+.. versionadded:: 1.2
+
+This also works with fixture groups.  If you create a subdirectory called
+``initial_data`` within a fixtures directory, then all fixture files in
+that subdirectory will be loaded each time you run :djadmin:`syncdb`.
+
 .. seealso::
 
     Fixtures are also used by the :ref:`testing framework

docs/ref/django-admin.txt

@@ -351 +351 @@
 
 The ``dumpdata`` command can be used to generate input for ``loaddata``.
 
+Fixture groups
+~~~~~~~~~~~~~~
+
+.. versionadded:: 1.2
+
+Fixtures can also be grouped together into subdirectories under the
+``fixtures`` directory.  The subdirectory can be referred to as a fixture, and
+:djadmin:`loaddata` will try to load each file in the subdirectory as a fixture
+file.
+
+For example, say you have an app called ``blog`` with two models, ``Entry`` and
+``Category``. If you want to separate your entries into multiple fixture
+files based on their category, you could create a subdirectory underneath your
+``blog`` app's ``fixtures`` directory called ``entries``.  Within
+``blog/fixtures/entries/`` you could create two fixture files, ``django.json``
+and ``python.json``.
+
+In this case, you can refer to the fixture group ``entries`` to install both
+fixtures at the same time.  :djadmin:`loaddata` will first try to find individual
+fixtures matching the name requested, and if it doesn't find any it will then
+try to look for a fixture group with that name. The command::
+
+    django-admin.py loaddata entries
+
+would first search for an individual fixture named ``entries``, and when it
+didn't find one it would look for a subdirectory with that name under each
+fixture directory.  When it found one, it would attempt to install each file
+within the subdirectory as a fixture.  Both ``django.json`` and ``python.json``
+would be loaded.
+
+If you wanted to install only one of the fixtures in the group, you could refer
+to it specifically with the command::
+
+    django-admin.py loaddata entries/django.json
+
+and it would load just the ``django.json`` file.
+
 Compressed fixtures
 ~~~~~~~~~~~~~~~~~~~