Ticket #6422: distinct_on.3.diff

AUTHORS

@@ -540 +540 @@
     Gasper Zejn <zejn@kiberpipa.org>
     Jarek Zgoda <jarek.zgoda@gmail.com>
     Cheng Zhang
+    Jeffrey Gelens <jeffrey@gelens.org>
 
 A big THANK YOU goes to:
 

django/db/backends/__init__.py

@@ -341 +341 @@
     supports_stddev = None
     can_introspect_foreign_keys = None
 
+    # Support for the DISTINCT ON clause
+    can_distinct_on_fields = False
+
     def __init__(self, connection):
         self.connection = connection
 
@@ -494 +497 @@
         """
         raise NotImplementedError('Full-text search is not implemented for this database backend')
 
+    def distinct(self, db_table, fields):
+        """
+        Returns an SQL DISTINCT clause which removes duplicate rows from the
+        result set. If any fields are given, only the given fields are being
+        checked for duplicates.
+        """
+        if fields:
+            raise NotImplementedError('DISTINCT ON fields is not supported by this database backend')
+        else:
+            return 'DISTINCT'
+
     def last_executed_query(self, cursor, sql, params):
         """
         Returns a string of the query last executed by the given cursor, with

django/db/backends/postgresql_psycopg2/base.py

@@ -71 +71 @@
     can_defer_constraint_checks = True
     has_select_for_update = True
     has_select_for_update_nowait = True
+    can_distinct_on_fields = True
 
 
 class DatabaseWrapper(BaseDatabaseWrapper):

django/db/backends/postgresql_psycopg2/operations.py

@@ -173 +173 @@
 
         return 63
 
+    def distinct(self, db_table, fields):
+        if fields:
+            table_name = self.quote_name(db_table)
+            fields = [table_name + "." + self.quote_name(field) for field in fields]
+            return 'DISTINCT ON (%s)' % ', '.join(fields)
+        else:
+            return 'DISTINCT'
+
     def last_executed_query(self, cursor, sql, params):
         # http://initd.org/psycopg/docs/cursor.html#cursor.query
         # The query attribute is a Psycopg extension to the DB API 2.0.

django/db/models/query.py

@@ -665 +665 @@
         obj.query.add_ordering(*field_names)
         return obj
 
-    def distinct(self, true_or_false=True):
+    def distinct(self, *field_names):
         """
         Returns a new QuerySet instance that will select only distinct results.
         """
         obj = self._clone()
-        obj.query.distinct = true_or_false
+        obj.query.add_distinct_fields(field_names)
+        obj.query.distinct = True
+
         return obj
 
     def extra(self, select=None, where=None, params=None, tables=None,
@@ -1090 +1092 @@
         """
         return self
 
-    def distinct(self, true_or_false=True):
+    def distinct(self, fields=None):
         """
         Always returns EmptyQuerySet.
         """

django/db/models/sql/compiler.py

@@ -74 +74 @@
             params.extend(val[1])
 
         result = ['SELECT']
+
         if self.query.distinct:
-            result.append('DISTINCT')
+            distinct_sql = self.connection.ops.distinct(
+                self.query.model._meta.db_table, self.query.distinct_fields)
+            result.append(distinct_sql)
+
         result.append(', '.join(out_cols + self.query.ordering_aliases))
 
         result.append('FROM')

django/db/models/sql/query.py

@@ -125 +125 @@
         self.order_by = []
         self.low_mark, self.high_mark = 0, None  # Used for offset/limit
         self.distinct = False
+        self.distinct_fields = None
         self.select_for_update = False
         self.select_for_update_nowait = False
         self.select_related = False
@@ -256 +257 @@
         obj.order_by = self.order_by[:]
         obj.low_mark, obj.high_mark = self.low_mark, self.high_mark
         obj.distinct = self.distinct
+        obj.distinct_fields = self.distinct_fields
         obj.select_for_update = self.select_for_update
         obj.select_for_update_nowait = self.select_for_update_nowait
         obj.select_related = self.select_related
@@ -384 +386 @@
         Performs a COUNT() query using the current filter constraints.
         """
         obj = self.clone()
-        if len(self.select) > 1 or self.aggregate_select:
+        if len(self.select) > 1 or self.aggregate_select or (self.distinct and self.distinct_fields):
             # If a select clause exists, then the query has already started to
             # specify the columns that are to be returned.
             # In this case, we need to use a subquery to evaluate the count.
@@ -1557 +1559 @@
         self.select = []
         self.select_fields = []
 
+    def add_distinct_fields(self, field_names):
+        self.distinct_fields = []
+        options = self.get_meta()
+
+        for name in field_names:
+            field, source, opts, join_list, last, _ = self.setup_joins(
+                name.split(LOOKUP_SEP), options, self.get_initial_alias(), False)
+            self.distinct_fields.append(field.column)
+
     def add_fields(self, field_names, allow_m2m=True):
         """
         Adds the given (model) fields to the select set. The field names are

docs/ref/models/querysets.txt

@@ -345 +345 @@
 distinct
 ~~~~~~~~
 
-.. method:: distinct()
+.. method:: distinct(*fields)
 
 Returns a new ``QuerySet`` that uses ``SELECT DISTINCT`` in its SQL query. This
 eliminates duplicate rows from the query results.
@@ -375 +375 @@
     ``values()`` together, be careful when ordering by fields not in the
     ``values()`` call.
 
+.. versionadded:: 1.4
+   ``distinct()`` takes optional positional arguments ``*fields``, which specify
+   field names to which the ``DISTINCT`` should be limited. This translates to
+   a ``SELECT DISTINCT ON`` SQL query. Note that this ``DISTINCT ON`` query is
+   only available in PostgreSQL.
+
+.. note::
+    When optional ``*fields`` are given, you will have to add an :meth:`order_by`
+    call with the same field names as the leftmost arguments.
+
 values
 ~~~~~~
 

tests/regressiontests/queries/models.py

@@ -208 +208 @@
     name = models.CharField("Name", max_length=20)
     greatest_fan = models.ForeignKey("Fan", null=True, unique=True)
 
+    def __unicode__(self):
+        return self.name
+
 class TvChef(Celebrity):
     pass
 
@@ -317 +320 @@
 
     def __unicode__(self):
        return self.name
+
+
+class Staff(models.Model):
+    name = models.CharField(max_length=50)
+    organisation = models.CharField(max_length=100)
+    tags = models.ManyToManyField(Tag, through='StaffTag')
+
+    def __unicode__(self):
+        return self.name
+
+class StaffTag(models.Model):
+    staff = models.ForeignKey(Staff)
+    tag = models.ForeignKey(Tag)
+
+    def __unicode__(self):
+        return u"%s -> %s" % (self.tag, self.staff)
+

tests/regressiontests/queries/tests.py

@@ -15 +15 @@
     DumbCategory, ExtraInfo, Fan, Item, LeafA, LoopX, LoopZ, ManagedModel,
     Member, NamedCategory, Note, Number, Plaything, PointerA, Ranking, Related,
     Report, ReservedName, Tag, TvChef, Valid, X, Food, Eaten, Node, ObjectA, ObjectB,
-    ObjectC)
+    ObjectC, Staff, StaffTag)
 
 
 class BaseQuerysetTest(TestCase):
@@ -1606 +1606 @@
         t4 = Tag.objects.create(name='t4', parent=t3)
         t5 = Tag.objects.create(name='t5', parent=t3)
 
+        p1_o1 = Staff.objects.create(name="p1", organisation="o1")
+        p2_o1 = Staff.objects.create(name="p2", organisation="o1")
+        p3_o1 = Staff.objects.create(name="p3", organisation="o1")
+        p1_o2 = Staff.objects.create(name="p1", organisation="o2")
+
+        StaffTag.objects.create(staff=p1_o1, tag=t1)
+        StaffTag.objects.create(staff=p1_o1, tag=t1)
+
+        celeb1 = Celebrity.objects.create(name="c1")
+        celeb2 = Celebrity.objects.create(name="c2")
+
+        self.fan1 = Fan.objects.create(fan_of=celeb1)
+        self.fan2 = Fan.objects.create(fan_of=celeb1)
+        self.fan3 = Fan.objects.create(fan_of=celeb2)
+
     # In Python 2.6 beta releases, exceptions raised in __len__ are swallowed
     # (Python issue 1242657), so these cases return an empty list, rather than
     # raising an exception. Not a lot we can do about that, unfortunately, due to
@@ -1677 +1692 @@
             2500
         )
 
+    @skipUnlessDBFeature('can_distinct_on_fields')
+    def test_ticket6422(self):
+        # (qset, expected) tuples
+        qsets = (
+            (
+                Staff.objects.distinct().order_by('name'),
+                ['<Staff: p1>', '<Staff: p1>', '<Staff: p2>', '<Staff: p3>'],
+            ),
+            (
+                Staff.objects.distinct('name').order_by('name'),
+                ['<Staff: p1>', '<Staff: p2>', '<Staff: p3>'],
+            ),
+            (
+                Staff.objects.distinct('organisation').order_by('organisation', 'name'),
+                ['<Staff: p1>', '<Staff: p1>'],
+            ),
+            (
+                Staff.objects.distinct('name', 'organisation').order_by('name', 'organisation'),
+                ['<Staff: p1>', '<Staff: p1>', '<Staff: p2>', '<Staff: p3>'],
+            ),
+            (
+                Celebrity.objects.filter(fan__in=[self.fan1, self.fan2, self.fan3]).\
+                    distinct('name').order_by('name'),
+                ['<Celebrity: c1>', '<Celebrity: c2>'],
+            ),
+            (
+                StaffTag.objects.distinct('staff','tag'),
+                ['<StaffTag: t1 -> p1>'],
+            ),
+            (
+                Tag.objects.order_by('parent__pk').distinct('parent'),
+                ['<Tag: t3>', '<Tag: t5>', '<Tag: t1>'],
+            )
+        )
+
+        for qset, expected in qsets:
+            self.assertQuerysetEqual(qset, expected)
+            self.assertEqual(qset.count(), len(expected))
+
+        # and check the fieldlookup
+        self.assertRaises(
+            FieldError,
+            lambda: Staff.objects.distinct('shrubbery')
+        )
+
+
 class UnionTests(unittest.TestCase):
     """
     Tests for the union of two querysets. Bug #12252.